You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2024/10/14 14:52:08 UTC
(iotdb) branch master updated: Load: Support loading table model TsFile into table model IoTDB (#13706)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 3b121f766e8 Load: Support loading table model TsFile into table model IoTDB (#13706)
3b121f766e8 is described below
commit 3b121f766e8177165bdb485053a4b91ab34407ed
Author: Zikun Ma <55...@users.noreply.github.com>
AuthorDate: Mon Oct 14 22:52:03 2024 +0800
Load: Support loading table model TsFile into table model IoTDB (#13706)
---
.../iotdb/db/protocol/session/SessionManager.java | 10 +
.../queryengine/plan/analyze/AnalyzeVisitor.java | 27 +-
.../plan/analyze/LoadTsFileAnalyzer.java | 936 ---------------------
.../analyze/load/LoadTsFileAnalyzeSchemaCache.java | 293 +++++++
.../plan/analyze/load/LoadTsFileAnalyzer.java | 224 +++++
.../load/LoadTsFileToTableModelAnalyzer.java | 241 ++++++
.../load/LoadTsFileToTreeModelAnalyzer.java | 173 ++++
.../analyze/load/SchemaAutoCreatorAndVerifier.java | 486 +++++++++++
.../execution/config/TableConfigTaskVisitor.java | 29 +-
.../plan/planner/LogicalPlanVisitor.java | 11 +-
.../plan/node/load/LoadSingleTsFileNode.java | 30 +-
.../planner/plan/node/load/LoadTsFileNode.java | 51 +-
.../relational/analyzer/StatementAnalyzer.java | 49 ++
.../plan/relational/metadata/TableSchema.java | 16 +
.../metadata/fetcher/TableDeviceSchemaFetcher.java | 2 +-
.../plan/relational/planner/RelationPlanner.java | 16 +
.../relational/planner/TableLogicalPlanner.java | 8 +
.../plan/relational/planner/TableModelPlanner.java | 37 +-
.../plan/relational/sql/ast/AstVisitor.java | 4 +
.../plan/relational/sql/ast/LoadTsFile.java | 176 ++++
.../plan/relational/sql/parser/AstBuilder.java | 37 +-
.../plan/relational/sql/util/SqlFormatter.java | 25 +
.../plan/relational/type/InternalTypeManager.java | 31 +
.../plan/scheduler/load/LoadTsFileScheduler.java | 34 +-
.../plan/statement/crud/LoadTsFileStatement.java | 42 +-
.../dataregion/memtable/TsFileProcessor.java | 5 +-
.../load/config/LoadTsFileConfigurator.java | 34 +
.../plan/planner/node/load/LoadTsFileNodeTest.java | 5 +-
.../iotdb/commons/partition/DataPartition.java | 2 +-
.../db/relational/grammar/sql/RelationalSql.g4 | 14 +-
30 files changed, 2052 insertions(+), 996 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java
index 2247a2d0bc7..e03aeb183f8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/session/SessionManager.java
@@ -383,6 +383,16 @@ public class SessionManager implements SessionManagerMBean {
session.getSqlDialect());
}
+ public SessionInfo getSessionInfoOfTableModel(IClientSession session) {
+ return new SessionInfo(
+ session.getId(),
+ session.getUsername(),
+ session.getZoneId(),
+ session.getClientVersion(),
+ session.getDatabaseName(),
+ IClientSession.SqlDialect.TABLE);
+ }
+
@Override
public Set<String> getAllRpcClients() {
return this.sessions.keySet().stream()
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
index f0a91e5e766..a98afe09f58 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
@@ -71,6 +71,9 @@ import org.apache.iotdb.db.queryengine.execution.operator.window.ainode.Inferenc
import org.apache.iotdb.db.queryengine.execution.operator.window.ainode.InferenceWindowType;
import org.apache.iotdb.db.queryengine.execution.operator.window.ainode.TailInferenceWindow;
import org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet;
+import org.apache.iotdb.db.queryengine.plan.analyze.load.LoadTsFileAnalyzer;
+import org.apache.iotdb.db.queryengine.plan.analyze.load.LoadTsFileToTableModelAnalyzer;
+import org.apache.iotdb.db.queryengine.plan.analyze.load.LoadTsFileToTreeModelAnalyzer;
import org.apache.iotdb.db.queryengine.plan.analyze.lock.DataNodeSchemaLockManager;
import org.apache.iotdb.db.queryengine.plan.analyze.lock.SchemaLockType;
import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher;
@@ -82,6 +85,7 @@ import org.apache.iotdb.db.queryengine.plan.expression.binary.CompareBinaryExpre
import org.apache.iotdb.db.queryengine.plan.expression.leaf.ConstantOperand;
import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand;
import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression;
+import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.MeasurementGroup;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.DeviceViewIntoPathDescriptor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.FillDescriptor;
@@ -157,6 +161,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement;
import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
import org.apache.iotdb.db.schemaengine.template.Template;
+import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator;
import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet;
import org.apache.iotdb.db.utils.constant.SqlConstant;
import org.apache.iotdb.rpc.RpcUtils;
@@ -3009,9 +3014,8 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
context.setQueryType(QueryType.WRITE);
final long startTime = System.nanoTime();
- try (final LoadTsFileAnalyzer loadTsfileAnalyzer =
- new LoadTsFileAnalyzer(loadTsFileStatement, context, partitionFetcher, schemaFetcher)) {
- return loadTsfileAnalyzer.analyzeFileByFile(loadTsFileStatement.isDeleteAfterLoad());
+ try (final LoadTsFileAnalyzer loadTsFileAnalyzer = getAnalyzer(loadTsFileStatement, context)) {
+ return (Analysis) loadTsFileAnalyzer.analyzeFileByFile(new Analysis());
} catch (final Exception e) {
final String exceptionMessage =
String.format(
@@ -3029,6 +3033,23 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
}
}
+ private LoadTsFileAnalyzer getAnalyzer(
+ LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) {
+ if (Objects.equals(loadTsFileStatement.getModel(), LoadTsFileConfigurator.MODEL_TREE_VALUE)) {
+ // Load to tree-model
+ return new LoadTsFileToTreeModelAnalyzer(loadTsFileStatement, context);
+ } else {
+ // Load to table-model
+ if (Objects.nonNull(loadTsFileStatement.getDatabase())) {
+ return new LoadTsFileToTableModelAnalyzer(
+ loadTsFileStatement, LocalExecutionPlanner.getInstance().metadata, context);
+ } else {
+ throw new SemanticException(
+ "Database name must be specified when loading data into the table model.");
+ }
+ }
+ }
+
private boolean analyzeTimeseriesRegionScan(
WhereCondition timeCondition,
PathPatternTree patternTree,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsFileAnalyzer.java
deleted file mode 100644
index c3482a3c1fe..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsFileAnalyzer.java
+++ /dev/null
@@ -1,936 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.analyze;
-
-import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.auth.AuthException;
-import org.apache.iotdb.commons.auth.entity.PrivilegeType;
-import org.apache.iotdb.commons.client.IClientManager;
-import org.apache.iotdb.commons.client.exception.ClientManagerException;
-import org.apache.iotdb.commons.conf.CommonDescriptor;
-import org.apache.iotdb.commons.consensus.ConfigRegionId;
-import org.apache.iotdb.commons.exception.IllegalPathException;
-import org.apache.iotdb.commons.path.MeasurementPath;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.commons.schema.SchemaConstant;
-import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics;
-import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp;
-import org.apache.iotdb.db.auth.AuthorityChecker;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.exception.LoadEmptyFileException;
-import org.apache.iotdb.db.exception.LoadFileException;
-import org.apache.iotdb.db.exception.LoadReadOnlyException;
-import org.apache.iotdb.db.exception.LoadRuntimeOutOfMemoryException;
-import org.apache.iotdb.db.exception.VerifyMetadataException;
-import org.apache.iotdb.db.exception.sql.SemanticException;
-import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
-import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
-import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
-import org.apache.iotdb.db.protocol.session.SessionManager;
-import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
-import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo;
-import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree;
-import org.apache.iotdb.db.queryengine.plan.Coordinator;
-import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher;
-import org.apache.iotdb.db.queryengine.plan.analyze.schema.SchemaValidator;
-import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult;
-import org.apache.iotdb.db.queryengine.plan.statement.Statement;
-import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.DatabaseSchemaStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDatabaseStatement;
-import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
-import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex;
-import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;
-import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils;
-import org.apache.iotdb.db.storageengine.load.memory.LoadTsFileAnalyzeSchemaMemoryBlock;
-import org.apache.iotdb.db.storageengine.load.memory.LoadTsFileMemoryManager;
-import org.apache.iotdb.db.utils.ModificationUtils;
-import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
-import org.apache.iotdb.db.utils.constant.SqlConstant;
-import org.apache.iotdb.rpc.RpcUtils;
-import org.apache.iotdb.rpc.TSStatusCode;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.thrift.TException;
-import org.apache.tsfile.common.constant.TsFileConstant;
-import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.file.metadata.IDeviceID;
-import org.apache.tsfile.file.metadata.TimeseriesMetadata;
-import org.apache.tsfile.file.metadata.enums.CompressionType;
-import org.apache.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.tsfile.read.TsFileSequenceReader;
-import org.apache.tsfile.read.TsFileSequenceReaderTimeseriesMetadataIterator;
-import org.apache.tsfile.utils.Pair;
-import org.apache.tsfile.write.schema.IMeasurementSchema;
-import org.apache.tsfile.write.schema.MeasurementSchema;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.BufferUnderflowException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
-public class LoadTsFileAnalyzer implements AutoCloseable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(LoadTsFileAnalyzer.class);
-
- private static final IClientManager<ConfigRegionId, ConfigNodeClient> CONFIG_NODE_CLIENT_MANAGER =
- ConfigNodeClientManager.getInstance();
- private static final int BATCH_FLUSH_TIME_SERIES_NUMBER;
- private static final int MAX_DEVICE_COUNT_TO_USE_DEVICE_TIME_INDEX;
- private static final long ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES;
- private static final long FLUSH_ALIGNED_CACHE_MEMORY_SIZE_IN_BYTES;
-
- static {
- final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
- BATCH_FLUSH_TIME_SERIES_NUMBER = CONFIG.getLoadTsFileAnalyzeSchemaBatchFlushTimeSeriesNumber();
- MAX_DEVICE_COUNT_TO_USE_DEVICE_TIME_INDEX =
- CONFIG.getLoadTsFileMaxDeviceCountToUseDeviceTimeIndex();
- ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES =
- CONFIG.getLoadTsFileAnalyzeSchemaMemorySizeInBytes() <= 0
- ? ((long) BATCH_FLUSH_TIME_SERIES_NUMBER) << 10
- : CONFIG.getLoadTsFileAnalyzeSchemaMemorySizeInBytes();
- FLUSH_ALIGNED_CACHE_MEMORY_SIZE_IN_BYTES = ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES >> 1;
- }
-
- private final LoadTsFileStatement loadTsFileStatement;
- private final MPPQueryContext context;
-
- private final IPartitionFetcher partitionFetcher;
- private final ISchemaFetcher schemaFetcher;
-
- private final SchemaAutoCreatorAndVerifier schemaAutoCreatorAndVerifier;
-
- LoadTsFileAnalyzer(
- LoadTsFileStatement loadTsFileStatement,
- MPPQueryContext context,
- IPartitionFetcher partitionFetcher,
- ISchemaFetcher schemaFetcher) {
- this.loadTsFileStatement = loadTsFileStatement;
- this.context = context;
-
- this.partitionFetcher = partitionFetcher;
- this.schemaFetcher = schemaFetcher;
-
- this.schemaAutoCreatorAndVerifier = new SchemaAutoCreatorAndVerifier();
- }
-
- public Analysis analyzeFileByFile(final boolean isDeleteAfterLoad) {
- final Analysis analysis = new Analysis();
-
- // check if the system is read only
- if (CommonDescriptor.getInstance().getConfig().isReadOnly()) {
- analysis.setFinishQueryAfterAnalyze(true);
- analysis.setFailStatus(
- RpcUtils.getStatus(TSStatusCode.SYSTEM_READ_ONLY, LoadReadOnlyException.MESSAGE));
- return analysis;
- }
-
- // analyze tsfile metadata file by file
- for (int i = 0, tsfileNum = loadTsFileStatement.getTsFiles().size(); i < tsfileNum; i++) {
- final File tsFile = loadTsFileStatement.getTsFiles().get(i);
-
- if (tsFile.length() == 0) {
- if (LOGGER.isWarnEnabled()) {
- LOGGER.warn("TsFile {} is empty.", tsFile.getPath());
- }
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(
- "Load - Analysis Stage: {}/{} tsfiles have been analyzed, progress: {}%",
- i + 1, tsfileNum, String.format("%.3f", (i + 1) * 100.00 / tsfileNum));
- }
- continue;
- }
-
- try {
- analyzeSingleTsFile(tsFile, isDeleteAfterLoad);
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(
- "Load - Analysis Stage: {}/{} tsfiles have been analyzed, progress: {}%",
- i + 1, tsfileNum, String.format("%.3f", (i + 1) * 100.00 / tsfileNum));
- }
- } catch (AuthException e) {
- return createFailAnalysisForAuthException(e);
- } catch (BufferUnderflowException e) {
- LOGGER.warn(
- "The file {} is not a valid tsfile. Please check the input file.", tsFile.getPath(), e);
- throw new SemanticException(
- String.format(
- "The file %s is not a valid tsfile. Please check the input file.",
- tsFile.getPath()));
- } catch (Exception e) {
- final String exceptionMessage =
- String.format(
- "The file %s is not a valid tsfile. Please check the input file. Detail: %s",
- tsFile.getPath(), e.getMessage() == null ? e.getClass().getName() : e.getMessage());
- LOGGER.warn(exceptionMessage, e);
- analysis.setFinishQueryAfterAnalyze(true);
- analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exceptionMessage));
- return analysis;
- }
- }
-
- try {
- schemaAutoCreatorAndVerifier.flush();
- } catch (AuthException e) {
- return createFailAnalysisForAuthException(e);
- } catch (Exception e) {
- final String exceptionMessage =
- String.format(
- "Auto create or verify schema error when executing statement %s. Detail: %s.",
- loadTsFileStatement,
- e.getMessage() == null ? e.getClass().getName() : e.getMessage());
- LOGGER.warn(exceptionMessage, e);
- analysis.setFinishQueryAfterAnalyze(true);
- analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exceptionMessage));
- return analysis;
- }
-
- LOGGER.info("Load - Analysis Stage: all tsfiles have been analyzed.");
-
- // data partition will be queried in the scheduler
- analysis.setRealStatement(loadTsFileStatement);
- return analysis;
- }
-
- @Override
- public void close() {
- schemaAutoCreatorAndVerifier.close();
- }
-
- private void analyzeSingleTsFile(final File tsFile, final boolean isDeleteAfterLoad)
- throws IOException, AuthException {
- try (final TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) {
- // can be reused when constructing tsfile resource
- final TsFileSequenceReaderTimeseriesMetadataIterator timeseriesMetadataIterator =
- new TsFileSequenceReaderTimeseriesMetadataIterator(reader, true, 1);
-
- // construct tsfile resource
- final TsFileResource tsFileResource = new TsFileResource(tsFile);
- if (!tsFileResource.resourceFileExists()) {
- // it will be serialized in LoadSingleTsFileNode
- tsFileResource.updatePlanIndexes(reader.getMinPlanIndex());
- tsFileResource.updatePlanIndexes(reader.getMaxPlanIndex());
- } else {
- tsFileResource.deserialize();
- }
-
- schemaAutoCreatorAndVerifier.setCurrentModificationsAndTimeIndex(tsFileResource, reader);
-
- // check if the tsfile is empty
- if (!timeseriesMetadataIterator.hasNext()) {
- throw new LoadEmptyFileException(tsFile.getAbsolutePath());
- }
-
- long writePointCount = 0;
-
- final boolean isAutoCreateSchemaOrVerifySchemaEnabled =
- IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled()
- || loadTsFileStatement.isVerifySchema();
- while (timeseriesMetadataIterator.hasNext()) {
- final Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadata =
- timeseriesMetadataIterator.next();
-
- if (isAutoCreateSchemaOrVerifySchemaEnabled) {
- schemaAutoCreatorAndVerifier.autoCreateAndVerify(reader, device2TimeseriesMetadata);
- }
-
- if (!tsFileResource.resourceFileExists()) {
- TsFileResourceUtils.updateTsFileResource(device2TimeseriesMetadata, tsFileResource);
- }
-
- // TODO: how to get the correct write point count when
- // !isAutoCreateSchemaOrVerifySchemaEnabled
- writePointCount += getWritePointCount(device2TimeseriesMetadata);
- }
- if (isAutoCreateSchemaOrVerifySchemaEnabled) {
- schemaAutoCreatorAndVerifier.flushAndClearDeviceIsAlignedCacheIfNecessary();
- }
-
- TimestampPrecisionUtils.checkTimestampPrecision(tsFileResource.getFileEndTime());
- tsFileResource.setStatus(TsFileResourceStatus.NORMAL);
-
- loadTsFileStatement.addTsFileResource(tsFileResource);
- loadTsFileStatement.addWritePointCount(writePointCount);
- } catch (final LoadEmptyFileException loadEmptyFileException) {
- LOGGER.warn("Failed to load empty file: {}", tsFile.getAbsolutePath());
- if (isDeleteAfterLoad) {
- FileUtils.deleteQuietly(tsFile);
- }
- }
- }
-
- private long getWritePointCount(
- Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadata) {
- return device2TimeseriesMetadata.values().stream()
- .flatMap(List::stream)
- .mapToLong(t -> t.getStatistics().getCount())
- .sum();
- }
-
- private Analysis createFailAnalysisForAuthException(AuthException e) {
- Analysis analysis = new Analysis();
- analysis.setFinishQueryAfterAnalyze(true);
- analysis.setFailStatus(RpcUtils.getStatus(e.getCode(), e.getMessage()));
- return analysis;
- }
-
- private final class SchemaAutoCreatorAndVerifier {
- private final LoadTsFileAnalyzeSchemaCache schemaCache;
-
- private SchemaAutoCreatorAndVerifier() throws LoadRuntimeOutOfMemoryException {
- this.schemaCache = new LoadTsFileAnalyzeSchemaCache();
- }
-
- public void setCurrentModificationsAndTimeIndex(
- TsFileResource resource, TsFileSequenceReader reader) throws IOException {
- schemaCache.setCurrentModificationsAndTimeIndex(resource, reader);
- }
-
- public void autoCreateAndVerify(
- TsFileSequenceReader reader,
- Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadataList)
- throws IOException, AuthException {
- for (final Map.Entry<IDeviceID, List<TimeseriesMetadata>> entry :
- device2TimeseriesMetadataList.entrySet()) {
- final IDeviceID device = entry.getKey();
-
- try {
- if (schemaCache.isDeviceDeletedByMods(device)) {
- continue;
- }
- } catch (IllegalPathException e) {
- LOGGER.warn(
- "Failed to check if device {} is deleted by mods. Will see it as not deleted.",
- device,
- e);
- }
-
- for (final TimeseriesMetadata timeseriesMetadata : entry.getValue()) {
- try {
- if (schemaCache.isTimeseriesDeletedByMods(device, timeseriesMetadata)) {
- continue;
- }
- } catch (IllegalPathException e) {
- LOGGER.warn(
- "Failed to check if device {}, timeseries {} is deleted by mods. Will see it as not deleted.",
- device,
- timeseriesMetadata.getMeasurementId(),
- e);
- }
-
- final TSDataType dataType = timeseriesMetadata.getTsDataType();
- if (TSDataType.VECTOR.equals(dataType)) {
- schemaCache
- .clearDeviceIsAlignedCacheIfNecessary(); // must execute before add aligned cache
- schemaCache.addIsAlignedCache(device, true, false);
-
- // not a timeseries, skip
- } else {
- // check WRITE_DATA permission of timeseries
- long startTime = System.nanoTime();
- try {
- String userName = context.getSession().getUserName();
- if (!AuthorityChecker.SUPER_USER.equals(userName)) {
- TSStatus status;
- try {
- List<PartialPath> paths =
- Collections.singletonList(
- new MeasurementPath(device, timeseriesMetadata.getMeasurementId()));
- status =
- AuthorityChecker.getTSStatus(
- AuthorityChecker.checkFullPathListPermission(
- userName, paths, PrivilegeType.WRITE_DATA.ordinal()),
- paths,
- PrivilegeType.WRITE_DATA);
- } catch (IllegalPathException e) {
- throw new RuntimeException(e);
- }
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- throw new AuthException(
- TSStatusCode.representOf(status.getCode()), status.getMessage());
- }
- }
- } finally {
- PerformanceOverviewMetrics.getInstance()
- .recordAuthCost(System.nanoTime() - startTime);
- }
- final Pair<CompressionType, TSEncoding> compressionEncodingPair =
- reader.readTimeseriesCompressionTypeAndEncoding(timeseriesMetadata);
- schemaCache.addTimeSeries(
- device,
- new MeasurementSchema(
- timeseriesMetadata.getMeasurementId(),
- dataType,
- compressionEncodingPair.getRight(),
- compressionEncodingPair.getLeft()));
-
- schemaCache.addIsAlignedCache(device, false, true);
- if (!schemaCache.getDeviceIsAligned(device)) {
- schemaCache.clearDeviceIsAlignedCacheIfNecessary();
- }
- }
-
- if (schemaCache.shouldFlushTimeSeries()) {
- flush();
- }
- }
- }
- }
-
- /**
- * This can only be invoked after all timeseries in the current tsfile have been processed.
- * Otherwise, the isAligned status may be wrong.
- */
- public void flushAndClearDeviceIsAlignedCacheIfNecessary() throws SemanticException {
- // avoid OOM when loading a tsfile with too many timeseries
- // or loading too many tsfiles at the same time
- schemaCache.clearDeviceIsAlignedCacheIfNecessary();
- }
-
- public void flush() throws AuthException {
- doAutoCreateAndVerify();
-
- schemaCache.clearTimeSeries();
- }
-
- private void doAutoCreateAndVerify() throws SemanticException, AuthException {
- if (schemaCache.getDevice2TimeSeries().isEmpty()) {
- return;
- }
-
- try {
- if (loadTsFileStatement.isVerifySchema()) {
- makeSureNoDuplicatedMeasurementsInDevices();
- }
-
- if (loadTsFileStatement.isAutoCreateDatabase()) {
- autoCreateDatabase();
- }
-
- // schema fetcher will not auto create if config set
- // isAutoCreateSchemaEnabled is false.
- final ISchemaTree schemaTree = autoCreateSchema();
-
- if (loadTsFileStatement.isVerifySchema()) {
- verifySchema(schemaTree);
- }
- } catch (AuthException e) {
- throw e;
- } catch (Exception e) {
- LOGGER.warn("Auto create or verify schema error.", e);
- throw new SemanticException(
- String.format(
- "Auto create or verify schema error when executing statement %s. Detail: %s.",
- loadTsFileStatement, e.getMessage()));
- }
- }
-
- private void makeSureNoDuplicatedMeasurementsInDevices() throws VerifyMetadataException {
- for (final Map.Entry<IDeviceID, Set<MeasurementSchema>> entry :
- schemaCache.getDevice2TimeSeries().entrySet()) {
- final IDeviceID device = entry.getKey();
- final Map<String, MeasurementSchema> measurement2Schema = new HashMap<>();
- for (final MeasurementSchema timeseriesSchema : entry.getValue()) {
- final String measurement = timeseriesSchema.getMeasurementId();
- if (measurement2Schema.containsKey(measurement)) {
- throw new VerifyMetadataException(
- String.format("Duplicated measurements %s in device %s.", measurement, device));
- }
- measurement2Schema.put(measurement, timeseriesSchema);
- }
- }
- }
-
- private void autoCreateDatabase()
- throws VerifyMetadataException, LoadFileException, IllegalPathException, AuthException {
- final int databasePrefixNodesLength = loadTsFileStatement.getDatabaseLevel() + 1;
- final Set<PartialPath> databasesNeededToBeSet = new HashSet<>();
-
- for (final IDeviceID device : schemaCache.getDevice2TimeSeries().keySet()) {
- final PartialPath devicePath = new PartialPath(device);
-
- final String[] devicePrefixNodes = devicePath.getNodes();
- if (devicePrefixNodes.length < databasePrefixNodesLength) {
- throw new VerifyMetadataException(
- String.format(
- "Database level %d is longer than device %s.",
- databasePrefixNodesLength, device));
- }
-
- final String[] databasePrefixNodes = new String[databasePrefixNodesLength];
- System.arraycopy(devicePrefixNodes, 0, databasePrefixNodes, 0, databasePrefixNodesLength);
-
- databasesNeededToBeSet.add(new PartialPath(databasePrefixNodes));
- }
-
- // 1. filter out the databases that already exist
- if (schemaCache.getAlreadySetDatabases().isEmpty()) {
- try (final ConfigNodeClient configNodeClient =
- CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
- final TGetDatabaseReq req =
- new TGetDatabaseReq(
- Arrays.asList(
- new ShowDatabaseStatement(new PartialPath(SqlConstant.getSingleRootArray()))
- .getPathPattern()
- .getNodes()),
- SchemaConstant.ALL_MATCH_SCOPE.serialize());
- final TShowDatabaseResp resp = configNodeClient.showDatabase(req);
-
- for (final String databaseName : resp.getDatabaseInfoMap().keySet()) {
- schemaCache.addAlreadySetDatabase(new PartialPath(databaseName));
- }
- } catch (IOException | TException | ClientManagerException e) {
- throw new LoadFileException(e);
- }
- }
- databasesNeededToBeSet.removeAll(schemaCache.getAlreadySetDatabases());
-
- // 2. create the databases that do not exist
- for (final PartialPath databasePath : databasesNeededToBeSet) {
- final DatabaseSchemaStatement statement =
- new DatabaseSchemaStatement(DatabaseSchemaStatement.DatabaseSchemaStatementType.CREATE);
- statement.setDatabasePath(databasePath);
- // do not print exception log because it is not an error
- statement.setEnablePrintExceptionLog(false);
- executeSetDatabaseStatement(statement);
-
- schemaCache.addAlreadySetDatabase(databasePath);
- }
- }
-
- private void executeSetDatabaseStatement(Statement statement)
- throws LoadFileException, AuthException {
- // 1.check Authority
- TSStatus status =
- AuthorityChecker.checkAuthority(statement, context.getSession().getUserName());
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- throw new AuthException(TSStatusCode.representOf(status.getCode()), status.getMessage());
- }
-
- // 2.execute setDatabase statement
- final long queryId = SessionManager.getInstance().requestQueryId();
- final ExecutionResult result =
- Coordinator.getInstance()
- .executeForTreeModel(
- statement,
- queryId,
- null,
- "",
- partitionFetcher,
- schemaFetcher,
- IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold());
- if (result.status.code != TSStatusCode.SUCCESS_STATUS.getStatusCode()
- && result.status.code != TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode()
- // In tree model, if the user creates a conflict database concurrently, for instance, the
- // database created by user is root.db.ss.a, the auto-creation failed database is root.db,
- // we wait till "getOrCreatePartition" to judge if the time series (like root.db.ss.a.e /
- // root.db.ss.a) conflicts with the created database. just do not throw exception here.
- && result.status.code != TSStatusCode.DATABASE_CONFLICT.getStatusCode()) {
- LOGGER.warn(
- "Create database error, statement: {}, result status is: {}", statement, result.status);
- throw new LoadFileException(
- String.format(
- "Create database error, statement: %s, result status is: %s",
- statement, result.status));
- }
- }
-
- private ISchemaTree autoCreateSchema() throws IllegalPathException {
- final List<PartialPath> deviceList = new ArrayList<>();
- final List<String[]> measurementList = new ArrayList<>();
- final List<TSDataType[]> dataTypeList = new ArrayList<>();
- final List<TSEncoding[]> encodingsList = new ArrayList<>();
- final List<CompressionType[]> compressionTypesList = new ArrayList<>();
- final List<Boolean> isAlignedList = new ArrayList<>();
-
- for (final Map.Entry<IDeviceID, Set<MeasurementSchema>> entry :
- schemaCache.getDevice2TimeSeries().entrySet()) {
- final int measurementSize = entry.getValue().size();
- final String[] measurements = new String[measurementSize];
- final TSDataType[] tsDataTypes = new TSDataType[measurementSize];
- final TSEncoding[] encodings = new TSEncoding[measurementSize];
- final CompressionType[] compressionTypes = new CompressionType[measurementSize];
-
- int index = 0;
- for (final MeasurementSchema measurementSchema : entry.getValue()) {
- measurements[index] = measurementSchema.getMeasurementId();
- tsDataTypes[index] = measurementSchema.getType();
- encodings[index] = measurementSchema.getEncodingType();
- compressionTypes[index++] = measurementSchema.getCompressor();
- }
-
- deviceList.add(new PartialPath(entry.getKey()));
- measurementList.add(measurements);
- dataTypeList.add(tsDataTypes);
- encodingsList.add(encodings);
- compressionTypesList.add(compressionTypes);
- isAlignedList.add(schemaCache.getDeviceIsAligned(entry.getKey()));
- }
-
- return SchemaValidator.validate(
- schemaFetcher,
- deviceList,
- measurementList,
- dataTypeList,
- encodingsList,
- compressionTypesList,
- isAlignedList,
- context);
- }
-
- private void verifySchema(ISchemaTree schemaTree)
- throws VerifyMetadataException, IllegalPathException {
- for (final Map.Entry<IDeviceID, Set<MeasurementSchema>> entry :
- schemaCache.getDevice2TimeSeries().entrySet()) {
- final IDeviceID device = entry.getKey();
- final List<IMeasurementSchema> tsfileTimeseriesSchemas = new ArrayList<>(entry.getValue());
- final DeviceSchemaInfo iotdbDeviceSchemaInfo =
- schemaTree.searchDeviceSchemaInfo(
- new PartialPath(device),
- tsfileTimeseriesSchemas.stream()
- .map(IMeasurementSchema::getMeasurementId)
- .collect(Collectors.toList()));
-
- if (iotdbDeviceSchemaInfo == null) {
- throw new VerifyMetadataException(
- String.format(
- "Device %s does not exist in IoTDB and can not be created. "
- + "Please check weather auto-create-schema is enabled.",
- device));
- }
-
- // check device schema: is aligned or not
- final boolean isAlignedInTsFile = schemaCache.getDeviceIsAligned(device);
- final boolean isAlignedInIoTDB = iotdbDeviceSchemaInfo.isAligned();
- if (isAlignedInTsFile != isAlignedInIoTDB) {
- throw new VerifyMetadataException(
- String.format(
- "Device %s in TsFile is %s, but in IoTDB is %s.",
- device,
- isAlignedInTsFile ? "aligned" : "not aligned",
- isAlignedInIoTDB ? "aligned" : "not aligned"));
- }
-
- // check timeseries schema
- final List<IMeasurementSchema> iotdbTimeseriesSchemas =
- iotdbDeviceSchemaInfo.getMeasurementSchemaList();
- for (int i = 0, n = iotdbTimeseriesSchemas.size(); i < n; i++) {
- final IMeasurementSchema tsFileSchema = tsfileTimeseriesSchemas.get(i);
- final IMeasurementSchema iotdbSchema = iotdbTimeseriesSchemas.get(i);
- if (iotdbSchema == null) {
- throw new VerifyMetadataException(
- String.format(
- "Measurement %s does not exist in IoTDB and can not be created. "
- + "Please check weather auto-create-schema is enabled.",
- device + TsFileConstant.PATH_SEPARATOR + tsfileTimeseriesSchemas.get(i)));
- }
-
- // check datatype
- if (!tsFileSchema.getType().equals(iotdbSchema.getType())) {
- throw new VerifyMetadataException(
- String.format(
- "Measurement %s%s%s datatype not match, TsFile: %s, IoTDB: %s",
- device,
- TsFileConstant.PATH_SEPARATOR,
- iotdbSchema.getMeasurementId(),
- tsFileSchema.getType(),
- iotdbSchema.getType()));
- }
-
- // check encoding
- if (LOGGER.isDebugEnabled()
- && !tsFileSchema.getEncodingType().equals(iotdbSchema.getEncodingType())) {
- // we allow a measurement to have different encodings in different chunks
- LOGGER.debug(
- "Encoding type not match, measurement: {}{}{}, "
- + "TsFile encoding: {}, IoTDB encoding: {}",
- device,
- TsFileConstant.PATH_SEPARATOR,
- iotdbSchema.getMeasurementId(),
- tsFileSchema.getEncodingType().name(),
- iotdbSchema.getEncodingType().name());
- }
-
- // check compressor
- if (LOGGER.isDebugEnabled()
- && !tsFileSchema.getCompressor().equals(iotdbSchema.getCompressor())) {
- // we allow a measurement to have different compressors in different chunks
- LOGGER.debug(
- "Compressor not match, measurement: {}{}{}, "
- + "TsFile compressor: {}, IoTDB compressor: {}",
- device,
- TsFileConstant.PATH_SEPARATOR,
- iotdbSchema.getMeasurementId(),
- tsFileSchema.getCompressor().name(),
- iotdbSchema.getCompressor().name());
- }
- }
- }
- }
-
- public void close() {
- schemaCache.close();
- }
- }
-
- private static class LoadTsFileAnalyzeSchemaCache {
-
- private final LoadTsFileAnalyzeSchemaMemoryBlock block;
-
- private Map<IDeviceID, Set<MeasurementSchema>> currentBatchDevice2TimeSeriesSchemas;
- private Map<IDeviceID, Boolean> tsFileDevice2IsAligned;
- private Set<PartialPath> alreadySetDatabases;
-
- private Collection<Modification> currentModifications;
- private ITimeIndex currentTimeIndex;
-
- private long batchDevice2TimeSeriesSchemasMemoryUsageSizeInBytes = 0;
- private long tsFileDevice2IsAlignedMemoryUsageSizeInBytes = 0;
- private long alreadySetDatabasesMemoryUsageSizeInBytes = 0;
- private long currentModificationsMemoryUsageSizeInBytes = 0;
- private long currentTimeIndexMemoryUsageSizeInBytes = 0;
-
- private int currentBatchTimeSeriesCount = 0;
-
- public LoadTsFileAnalyzeSchemaCache() throws LoadRuntimeOutOfMemoryException {
- this.block =
- LoadTsFileMemoryManager.getInstance()
- .allocateAnalyzeSchemaMemoryBlock(ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES);
- this.currentBatchDevice2TimeSeriesSchemas = new HashMap<>();
- this.tsFileDevice2IsAligned = new HashMap<>();
- this.alreadySetDatabases = new HashSet<>();
- this.currentModifications = new ArrayList<>();
- }
-
- public Map<IDeviceID, Set<MeasurementSchema>> getDevice2TimeSeries() {
- return currentBatchDevice2TimeSeriesSchemas;
- }
-
- public boolean getDeviceIsAligned(IDeviceID device) {
- if (!tsFileDevice2IsAligned.containsKey(device)) {
- LOGGER.warn(
- "Device {} is not in the tsFileDevice2IsAligned cache {}.",
- device,
- tsFileDevice2IsAligned);
- }
- return tsFileDevice2IsAligned.get(device);
- }
-
- public Set<PartialPath> getAlreadySetDatabases() {
- return alreadySetDatabases;
- }
-
- public void addTimeSeries(IDeviceID device, MeasurementSchema measurementSchema) {
- long memoryUsageSizeInBytes = 0;
- if (!currentBatchDevice2TimeSeriesSchemas.containsKey(device)) {
- memoryUsageSizeInBytes += device.ramBytesUsed();
- }
- if (currentBatchDevice2TimeSeriesSchemas
- .computeIfAbsent(device, k -> new HashSet<>())
- .add(measurementSchema)) {
- memoryUsageSizeInBytes += measurementSchema.serializedSize();
- currentBatchTimeSeriesCount++;
- }
-
- if (memoryUsageSizeInBytes > 0) {
- batchDevice2TimeSeriesSchemasMemoryUsageSizeInBytes += memoryUsageSizeInBytes;
- block.addMemoryUsage(memoryUsageSizeInBytes);
- }
- }
-
- public void addIsAlignedCache(IDeviceID device, boolean isAligned, boolean addIfAbsent) {
- long memoryUsageSizeInBytes = 0;
- if (!tsFileDevice2IsAligned.containsKey(device)) {
- memoryUsageSizeInBytes += device.ramBytesUsed();
- }
- if (addIfAbsent
- ? (tsFileDevice2IsAligned.putIfAbsent(device, isAligned) == null)
- : (tsFileDevice2IsAligned.put(device, isAligned) == null)) {
- memoryUsageSizeInBytes += Byte.BYTES;
- }
-
- if (memoryUsageSizeInBytes > 0) {
- tsFileDevice2IsAlignedMemoryUsageSizeInBytes += memoryUsageSizeInBytes;
- block.addMemoryUsage(memoryUsageSizeInBytes);
- }
- }
-
- public void setCurrentModificationsAndTimeIndex(
- TsFileResource resource, TsFileSequenceReader reader) throws IOException {
- clearModificationsAndTimeIndex();
-
- currentModifications = resource.getModFile().getModifications();
- for (final Modification modification : currentModifications) {
- currentModificationsMemoryUsageSizeInBytes += ((Deletion) modification).getSerializedSize();
- }
-
- // If there are too many modifications, a larger memory block is needed to avoid frequent
- // flush.
- long newMemorySize =
- currentModificationsMemoryUsageSizeInBytes > ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES / 2
- ? currentModificationsMemoryUsageSizeInBytes + ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES
- : ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES;
- block.forceResize(newMemorySize);
- block.addMemoryUsage(currentModificationsMemoryUsageSizeInBytes);
-
- // No need to build device time index if there are no modifications
- if (currentModifications.size() > 0 && resource.resourceFileExists()) {
- final AtomicInteger deviceCount = new AtomicInteger();
- reader
- .getAllDevicesIteratorWithIsAligned()
- .forEachRemaining(o -> deviceCount.getAndIncrement());
-
- // Use device time index only if the device count is less than the threshold, avoiding too
- // much memory usage
- if (deviceCount.get() < MAX_DEVICE_COUNT_TO_USE_DEVICE_TIME_INDEX) {
- currentTimeIndex = resource.getTimeIndex();
- if (currentTimeIndex instanceof FileTimeIndex) {
- currentTimeIndex = resource.buildDeviceTimeIndex();
- }
- currentTimeIndexMemoryUsageSizeInBytes = currentTimeIndex.calculateRamSize();
- block.addMemoryUsage(currentTimeIndexMemoryUsageSizeInBytes);
- }
- }
- }
-
- public boolean isDeviceDeletedByMods(IDeviceID device) throws IllegalPathException {
- return currentTimeIndex != null
- && ModificationUtils.isAllDeletedByMods(
- currentModifications,
- device,
- currentTimeIndex.getStartTime(device),
- currentTimeIndex.getEndTime(device));
- }
-
- public boolean isTimeseriesDeletedByMods(
- IDeviceID device, TimeseriesMetadata timeseriesMetadata) throws IllegalPathException {
- return ModificationUtils.isTimeseriesDeletedByMods(
- currentModifications,
- device,
- timeseriesMetadata.getMeasurementId(),
- timeseriesMetadata.getStatistics().getStartTime(),
- timeseriesMetadata.getStatistics().getEndTime());
- }
-
- public void addAlreadySetDatabase(PartialPath database) {
- long memoryUsageSizeInBytes = 0;
- if (alreadySetDatabases.add(database)) {
- memoryUsageSizeInBytes += PartialPath.estimateSize(database);
- }
-
- if (memoryUsageSizeInBytes > 0) {
- alreadySetDatabasesMemoryUsageSizeInBytes += memoryUsageSizeInBytes;
- block.addMemoryUsage(memoryUsageSizeInBytes);
- }
- }
-
- public boolean shouldFlushTimeSeries() {
- return !block.hasEnoughMemory()
- || currentBatchTimeSeriesCount >= BATCH_FLUSH_TIME_SERIES_NUMBER;
- }
-
- public boolean shouldFlushAlignedCache() {
- return tsFileDevice2IsAlignedMemoryUsageSizeInBytes
- >= FLUSH_ALIGNED_CACHE_MEMORY_SIZE_IN_BYTES;
- }
-
- public void clearTimeSeries() {
- currentBatchDevice2TimeSeriesSchemas.clear();
- block.reduceMemoryUsage(batchDevice2TimeSeriesSchemasMemoryUsageSizeInBytes);
- batchDevice2TimeSeriesSchemasMemoryUsageSizeInBytes = 0;
- currentBatchTimeSeriesCount = 0;
- }
-
- public void clearModificationsAndTimeIndex() {
- currentModifications.clear();
- currentTimeIndex = null;
- block.reduceMemoryUsage(currentModificationsMemoryUsageSizeInBytes);
- block.reduceMemoryUsage(currentTimeIndexMemoryUsageSizeInBytes);
- currentModificationsMemoryUsageSizeInBytes = 0;
- currentTimeIndexMemoryUsageSizeInBytes = 0;
- }
-
- public void clearAlignedCache() {
- tsFileDevice2IsAligned.clear();
- block.reduceMemoryUsage(tsFileDevice2IsAlignedMemoryUsageSizeInBytes);
- tsFileDevice2IsAlignedMemoryUsageSizeInBytes = 0;
- }
-
- public void clearDeviceIsAlignedCacheIfNecessary() {
- if (!shouldFlushAlignedCache()) {
- return;
- }
-
- long releaseMemoryInBytes = 0;
- final Set<IDeviceID> timeSeriesCacheKeySet =
- new HashSet<>(currentBatchDevice2TimeSeriesSchemas.keySet());
- Iterator<Map.Entry<IDeviceID, Boolean>> iterator =
- tsFileDevice2IsAligned.entrySet().iterator();
- while (iterator.hasNext()) {
- Map.Entry<IDeviceID, Boolean> entry = iterator.next();
- if (!timeSeriesCacheKeySet.contains(entry.getKey())) {
- releaseMemoryInBytes += entry.getKey().ramBytesUsed() + Byte.BYTES;
- iterator.remove();
- }
- }
- if (releaseMemoryInBytes > 0) {
- tsFileDevice2IsAlignedMemoryUsageSizeInBytes -= releaseMemoryInBytes;
- block.reduceMemoryUsage(releaseMemoryInBytes);
- }
- }
-
- private void clearDatabasesCache() {
- alreadySetDatabases.clear();
- block.reduceMemoryUsage(alreadySetDatabasesMemoryUsageSizeInBytes);
- alreadySetDatabasesMemoryUsageSizeInBytes = 0;
- }
-
- public void close() {
- clearTimeSeries();
- clearModificationsAndTimeIndex();
- clearAlignedCache();
- clearDatabasesCache();
-
- block.close();
-
- currentBatchDevice2TimeSeriesSchemas = null;
- tsFileDevice2IsAligned = null;
- alreadySetDatabases = null;
- }
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzeSchemaCache.java
new file mode 100644
index 00000000000..b0e4d63adcc
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzeSchemaCache.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.analyze.load;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.LoadRuntimeOutOfMemoryException;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;
+import org.apache.iotdb.db.storageengine.load.memory.LoadTsFileAnalyzeSchemaMemoryBlock;
+import org.apache.iotdb.db.storageengine.load.memory.LoadTsFileMemoryManager;
+import org.apache.iotdb.db.utils.ModificationUtils;
+
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class LoadTsFileAnalyzeSchemaCache {
+ private static final Logger LOGGER = LoggerFactory.getLogger(LoadTsFileAnalyzeSchemaCache.class);
+
+ private static final int BATCH_FLUSH_TIME_SERIES_NUMBER;
+ private static final int MAX_DEVICE_COUNT_TO_USE_DEVICE_TIME_INDEX;
+ private static final long ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES;
+ private static final long FLUSH_ALIGNED_CACHE_MEMORY_SIZE_IN_BYTES;
+
+ static {
+ final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
+ BATCH_FLUSH_TIME_SERIES_NUMBER = CONFIG.getLoadTsFileAnalyzeSchemaBatchFlushTimeSeriesNumber();
+ MAX_DEVICE_COUNT_TO_USE_DEVICE_TIME_INDEX =
+ CONFIG.getLoadTsFileMaxDeviceCountToUseDeviceTimeIndex();
+ ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES =
+ CONFIG.getLoadTsFileAnalyzeSchemaMemorySizeInBytes() <= 0
+ ? ((long) BATCH_FLUSH_TIME_SERIES_NUMBER) << 10
+ : CONFIG.getLoadTsFileAnalyzeSchemaMemorySizeInBytes();
+ FLUSH_ALIGNED_CACHE_MEMORY_SIZE_IN_BYTES = ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES >> 1;
+ }
+
+ private final LoadTsFileAnalyzeSchemaMemoryBlock block;
+
+ private Map<IDeviceID, Set<MeasurementSchema>> currentBatchDevice2TimeSeriesSchemas;
+ private Map<IDeviceID, Boolean> tsFileDevice2IsAligned;
+ private Set<PartialPath> alreadySetDatabases;
+
+ private Collection<Modification> currentModifications;
+ private ITimeIndex currentTimeIndex;
+
+ private long batchDevice2TimeSeriesSchemasMemoryUsageSizeInBytes = 0;
+ private long tsFileDevice2IsAlignedMemoryUsageSizeInBytes = 0;
+ private long alreadySetDatabasesMemoryUsageSizeInBytes = 0;
+ private long currentModificationsMemoryUsageSizeInBytes = 0;
+ private long currentTimeIndexMemoryUsageSizeInBytes = 0;
+
+ private int currentBatchTimeSeriesCount = 0;
+
+ public LoadTsFileAnalyzeSchemaCache() throws LoadRuntimeOutOfMemoryException {
+ this.block =
+ LoadTsFileMemoryManager.getInstance()
+ .allocateAnalyzeSchemaMemoryBlock(ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES);
+ this.currentBatchDevice2TimeSeriesSchemas = new HashMap<>();
+ this.tsFileDevice2IsAligned = new HashMap<>();
+ this.alreadySetDatabases = new HashSet<>();
+ this.currentModifications = new ArrayList<>();
+ }
+
+ public Map<IDeviceID, Set<MeasurementSchema>> getDevice2TimeSeries() {
+ return currentBatchDevice2TimeSeriesSchemas;
+ }
+
+ public boolean getDeviceIsAligned(IDeviceID device) {
+ if (!tsFileDevice2IsAligned.containsKey(device)) {
+ LOGGER.warn(
+ "Device {} is not in the tsFileDevice2IsAligned cache {}.",
+ device,
+ tsFileDevice2IsAligned);
+ }
+ return tsFileDevice2IsAligned.get(device);
+ }
+
+ public Set<PartialPath> getAlreadySetDatabases() {
+ return alreadySetDatabases;
+ }
+
+ public void addTimeSeries(IDeviceID device, MeasurementSchema measurementSchema) {
+ long memoryUsageSizeInBytes = 0;
+ if (!currentBatchDevice2TimeSeriesSchemas.containsKey(device)) {
+ memoryUsageSizeInBytes += device.ramBytesUsed();
+ }
+ if (currentBatchDevice2TimeSeriesSchemas
+ .computeIfAbsent(device, k -> new HashSet<>())
+ .add(measurementSchema)) {
+ memoryUsageSizeInBytes += measurementSchema.serializedSize();
+ currentBatchTimeSeriesCount++;
+ }
+
+ if (memoryUsageSizeInBytes > 0) {
+ batchDevice2TimeSeriesSchemasMemoryUsageSizeInBytes += memoryUsageSizeInBytes;
+ block.addMemoryUsage(memoryUsageSizeInBytes);
+ }
+ }
+
+ public void addIsAlignedCache(IDeviceID device, boolean isAligned, boolean addIfAbsent) {
+ long memoryUsageSizeInBytes = 0;
+ if (!tsFileDevice2IsAligned.containsKey(device)) {
+ memoryUsageSizeInBytes += device.ramBytesUsed();
+ }
+ if (addIfAbsent
+ ? (tsFileDevice2IsAligned.putIfAbsent(device, isAligned) == null)
+ : (tsFileDevice2IsAligned.put(device, isAligned) == null)) {
+ memoryUsageSizeInBytes += Byte.BYTES;
+ }
+
+ if (memoryUsageSizeInBytes > 0) {
+ tsFileDevice2IsAlignedMemoryUsageSizeInBytes += memoryUsageSizeInBytes;
+ block.addMemoryUsage(memoryUsageSizeInBytes);
+ }
+ }
+
+ public void setCurrentModificationsAndTimeIndex(
+ TsFileResource resource, TsFileSequenceReader reader) throws IOException {
+ clearModificationsAndTimeIndex();
+
+ currentModifications = resource.getModFile().getModifications();
+ for (final Modification modification : currentModifications) {
+ currentModificationsMemoryUsageSizeInBytes += ((Deletion) modification).getSerializedSize();
+ }
+
+ // If there are too many modifications, a larger memory block is needed to avoid frequent
+ // flush.
+ long newMemorySize =
+ currentModificationsMemoryUsageSizeInBytes > ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES / 2
+ ? currentModificationsMemoryUsageSizeInBytes + ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES
+ : ANALYZE_SCHEMA_MEMORY_SIZE_IN_BYTES;
+ block.forceResize(newMemorySize);
+ block.addMemoryUsage(currentModificationsMemoryUsageSizeInBytes);
+
+ // No need to build device time index if there are no modifications
+ if (!currentModifications.isEmpty() && resource.resourceFileExists()) {
+ final AtomicInteger deviceCount = new AtomicInteger();
+ reader
+ .getAllDevicesIteratorWithIsAligned()
+ .forEachRemaining(o -> deviceCount.getAndIncrement());
+
+ // Use device time index only if the device count is less than the threshold, avoiding too
+ // much memory usage
+ if (deviceCount.get() < MAX_DEVICE_COUNT_TO_USE_DEVICE_TIME_INDEX) {
+ currentTimeIndex = resource.getTimeIndex();
+ if (currentTimeIndex instanceof FileTimeIndex) {
+ currentTimeIndex = resource.buildDeviceTimeIndex();
+ }
+ currentTimeIndexMemoryUsageSizeInBytes = currentTimeIndex.calculateRamSize();
+ block.addMemoryUsage(currentTimeIndexMemoryUsageSizeInBytes);
+ }
+ }
+ }
+
+ public boolean isDeviceDeletedByMods(IDeviceID device) throws IllegalPathException {
+ return currentTimeIndex != null
+ && ModificationUtils.isAllDeletedByMods(
+ currentModifications,
+ device,
+ currentTimeIndex.getStartTime(device),
+ currentTimeIndex.getEndTime(device));
+ }
+
+ public boolean isTimeseriesDeletedByMods(IDeviceID device, TimeseriesMetadata timeseriesMetadata)
+ throws IllegalPathException {
+ return ModificationUtils.isTimeseriesDeletedByMods(
+ currentModifications,
+ device,
+ timeseriesMetadata.getMeasurementId(),
+ timeseriesMetadata.getStatistics().getStartTime(),
+ timeseriesMetadata.getStatistics().getEndTime());
+ }
+
+ public void addAlreadySetDatabase(PartialPath database) {
+ long memoryUsageSizeInBytes = 0;
+ if (alreadySetDatabases.add(database)) {
+ memoryUsageSizeInBytes += PartialPath.estimateSize(database);
+ }
+
+ if (memoryUsageSizeInBytes > 0) {
+ alreadySetDatabasesMemoryUsageSizeInBytes += memoryUsageSizeInBytes;
+ block.addMemoryUsage(memoryUsageSizeInBytes);
+ }
+ }
+
+ public boolean shouldFlushTimeSeries() {
+ return !block.hasEnoughMemory()
+ || currentBatchTimeSeriesCount >= BATCH_FLUSH_TIME_SERIES_NUMBER;
+ }
+
+ public boolean shouldFlushAlignedCache() {
+ return tsFileDevice2IsAlignedMemoryUsageSizeInBytes >= FLUSH_ALIGNED_CACHE_MEMORY_SIZE_IN_BYTES;
+ }
+
+ public void clearTimeSeries() {
+ currentBatchDevice2TimeSeriesSchemas.clear();
+ block.reduceMemoryUsage(batchDevice2TimeSeriesSchemasMemoryUsageSizeInBytes);
+ batchDevice2TimeSeriesSchemasMemoryUsageSizeInBytes = 0;
+ currentBatchTimeSeriesCount = 0;
+ }
+
+ public void clearModificationsAndTimeIndex() {
+ currentModifications.clear();
+ currentTimeIndex = null;
+ block.reduceMemoryUsage(currentModificationsMemoryUsageSizeInBytes);
+ block.reduceMemoryUsage(currentTimeIndexMemoryUsageSizeInBytes);
+ currentModificationsMemoryUsageSizeInBytes = 0;
+ currentTimeIndexMemoryUsageSizeInBytes = 0;
+ }
+
+ public void clearAlignedCache() {
+ tsFileDevice2IsAligned.clear();
+ block.reduceMemoryUsage(tsFileDevice2IsAlignedMemoryUsageSizeInBytes);
+ tsFileDevice2IsAlignedMemoryUsageSizeInBytes = 0;
+ }
+
+ public void clearDeviceIsAlignedCacheIfNecessary() {
+ if (!shouldFlushAlignedCache()) {
+ return;
+ }
+
+ long releaseMemoryInBytes = 0;
+ final Set<IDeviceID> timeSeriesCacheKeySet =
+ new HashSet<>(currentBatchDevice2TimeSeriesSchemas.keySet());
+ Iterator<Map.Entry<IDeviceID, Boolean>> iterator = tsFileDevice2IsAligned.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Map.Entry<IDeviceID, Boolean> entry = iterator.next();
+ if (!timeSeriesCacheKeySet.contains(entry.getKey())) {
+ releaseMemoryInBytes += entry.getKey().ramBytesUsed() + Byte.BYTES;
+ iterator.remove();
+ }
+ }
+ if (releaseMemoryInBytes > 0) {
+ tsFileDevice2IsAlignedMemoryUsageSizeInBytes -= releaseMemoryInBytes;
+ block.reduceMemoryUsage(releaseMemoryInBytes);
+ }
+ }
+
+ private void clearDatabasesCache() {
+ alreadySetDatabases.clear();
+ block.reduceMemoryUsage(alreadySetDatabasesMemoryUsageSizeInBytes);
+ alreadySetDatabasesMemoryUsageSizeInBytes = 0;
+ }
+
+ public void close() {
+ clearTimeSeries();
+ clearModificationsAndTimeIndex();
+ clearAlignedCache();
+ clearDatabasesCache();
+
+ block.close();
+
+ currentBatchDevice2TimeSeriesSchemas = null;
+ tsFileDevice2IsAligned = null;
+ alreadySetDatabases = null;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
new file mode 100644
index 00000000000..6e965c80b01
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.analyze.load;
+
+import org.apache.iotdb.commons.auth.AuthException;
+import org.apache.iotdb.db.exception.sql.SemanticException;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher;
+import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
+import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher;
+import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher;
+import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.BufferUnderflowException;
+import java.util.List;
+import java.util.Map;
+
+public abstract class LoadTsFileAnalyzer implements AutoCloseable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(LoadTsFileAnalyzer.class);
+
+ // These are only used when constructed from tree model SQL
+ private final LoadTsFileStatement loadTsFileStatement;
+
+ // These are only used when constructed from table model SQL
+ private final LoadTsFile loadTsFileTableStatement;
+
+ private final boolean isTableModelStatement;
+
+ protected final List<File> tsFiles;
+ protected final String statementString;
+ protected final boolean isVerifySchema;
+
+ protected final boolean isDeleteAfterLoad;
+
+ protected final boolean isAutoCreateDatabase;
+
+ protected final int databaseLevel;
+
+ protected final String database;
+
+ final MPPQueryContext context;
+
+ final IPartitionFetcher partitionFetcher = ClusterPartitionFetcher.getInstance();
+ final ISchemaFetcher schemaFetcher = ClusterSchemaFetcher.getInstance();
+
+ LoadTsFileAnalyzer(LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) {
+ this.loadTsFileStatement = loadTsFileStatement;
+ this.tsFiles = loadTsFileStatement.getTsFiles();
+ this.statementString = loadTsFileStatement.toString();
+ this.isVerifySchema = loadTsFileStatement.isVerifySchema();
+ this.isDeleteAfterLoad = loadTsFileStatement.isDeleteAfterLoad();
+ this.isAutoCreateDatabase = loadTsFileStatement.isAutoCreateDatabase();
+ this.databaseLevel = loadTsFileStatement.getDatabaseLevel();
+ this.database = loadTsFileStatement.getDatabase();
+
+ this.loadTsFileTableStatement = null;
+ this.isTableModelStatement = false;
+ this.context = context;
+ }
+
+ LoadTsFileAnalyzer(LoadTsFile loadTsFileTableStatement, MPPQueryContext context) {
+ this.loadTsFileTableStatement = loadTsFileTableStatement;
+ this.tsFiles = loadTsFileTableStatement.getTsFiles();
+ this.statementString = loadTsFileTableStatement.toString();
+ this.isVerifySchema = true;
+ this.isDeleteAfterLoad = loadTsFileTableStatement.isDeleteAfterLoad();
+ this.isAutoCreateDatabase = loadTsFileTableStatement.isAutoCreateDatabase();
+ this.databaseLevel = loadTsFileTableStatement.getDatabaseLevel();
+ this.database = loadTsFileTableStatement.getDatabase();
+
+ this.loadTsFileStatement = null;
+ this.isTableModelStatement = true;
+ this.context = context;
+ }
+
+ public abstract IAnalysis analyzeFileByFile(IAnalysis analysis);
+
+ protected boolean doAnalyzeFileByFile(IAnalysis analysis) {
+ // analyze tsfile metadata file by file
+ for (int i = 0, tsfileNum = tsFiles.size(); i < tsfileNum; i++) {
+ final File tsFile = tsFiles.get(i);
+
+ if (tsFile.length() == 0) {
+ if (LOGGER.isWarnEnabled()) {
+ LOGGER.warn("TsFile {} is empty.", tsFile.getPath());
+ }
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info(
+ "Load - Analysis Stage: {}/{} tsfiles have been analyzed, progress: {}%",
+ i + 1, tsfileNum, String.format("%.3f", (i + 1) * 100.00 / tsfileNum));
+ }
+ continue;
+ }
+
+ try {
+ analyzeSingleTsFile(tsFile);
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info(
+ "Load - Analysis Stage: {}/{} tsfiles have been analyzed, progress: {}%",
+ i + 1, tsfileNum, String.format("%.3f", (i + 1) * 100.00 / tsfileNum));
+ }
+ } catch (AuthException e) {
+ setFailAnalysisForAuthException(analysis, e);
+ return false;
+ } catch (BufferUnderflowException e) {
+ LOGGER.warn(
+ "The file {} is not a valid tsfile. Please check the input file.", tsFile.getPath(), e);
+ throw new SemanticException(
+ String.format(
+ "The file %s is not a valid tsfile. Please check the input file.",
+ tsFile.getPath()));
+ } catch (Exception e) {
+ final String exceptionMessage =
+ String.format(
+ "The file %s is not a valid tsfile. Please check the input file. Detail: %s",
+ tsFile.getPath(), e.getMessage() == null ? e.getClass().getName() : e.getMessage());
+ LOGGER.warn(exceptionMessage, e);
+ analysis.setFinishQueryAfterAnalyze(true);
+ analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exceptionMessage));
+ return false;
+ }
+ }
+ return true;
+ }
+
+ protected abstract void analyzeSingleTsFile(final File tsFile) throws IOException, AuthException;
+
+ protected TsFileResource constructTsFileResource(
+ final TsFileSequenceReader reader, final File tsFile) throws IOException {
+ final TsFileResource tsFileResource = new TsFileResource(tsFile);
+ if (!tsFileResource.resourceFileExists()) {
+ // it will be serialized in LoadSingleTsFileNode
+ tsFileResource.updatePlanIndexes(reader.getMinPlanIndex());
+ tsFileResource.updatePlanIndexes(reader.getMaxPlanIndex());
+ } else {
+ tsFileResource.deserialize();
+ }
+ return tsFileResource;
+ }
+
+ protected String getStatementString() {
+ return statementString;
+ }
+
+ protected void setRealStatement(IAnalysis analysis) {
+ if (isTableModelStatement) {
+ // Do nothing by now.
+ } else {
+ analysis.setRealStatement(loadTsFileStatement);
+ }
+ }
+
+ protected void addTsFileResource(TsFileResource tsFileResource) {
+ if (isTableModelStatement) {
+ loadTsFileTableStatement.addTsFileResource(tsFileResource);
+ } else {
+ loadTsFileStatement.addTsFileResource(tsFileResource);
+ }
+ }
+
+ protected void addWritePointCount(long writePointCount) {
+ if (isTableModelStatement) {
+ loadTsFileTableStatement.addWritePointCount(writePointCount);
+ } else {
+ loadTsFileStatement.addWritePointCount(writePointCount);
+ }
+ }
+
+ protected boolean isVerifySchema() {
+ return isVerifySchema;
+ }
+
+ protected boolean isAutoCreateDatabase() {
+ return isAutoCreateDatabase;
+ }
+
+ protected int getDatabaseLevel() {
+ return databaseLevel;
+ }
+
+ protected long getWritePointCount(
+ Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadata) {
+ return device2TimeseriesMetadata.values().stream()
+ .flatMap(List::stream)
+ .mapToLong(t -> t.getStatistics().getCount())
+ .sum();
+ }
+
+ protected void setFailAnalysisForAuthException(IAnalysis analysis, AuthException e) {
+ analysis.setFinishQueryAfterAnalyze(true);
+ analysis.setFailStatus(RpcUtils.getStatus(e.getCode(), e.getMessage()));
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java
new file mode 100644
index 00000000000..7316e7a3de1
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTableModelAnalyzer.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.analyze.load;
+
+import org.apache.iotdb.commons.auth.AuthException;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema;
+import org.apache.iotdb.db.exception.LoadEmptyFileException;
+import org.apache.iotdb.db.exception.LoadReadOnlyException;
+import org.apache.iotdb.db.exception.VerifyMetadataException;
+import org.apache.iotdb.db.exception.sql.SemanticException;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
+import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
+import org.apache.iotdb.db.queryengine.plan.execution.config.executor.ClusterConfigTaskExecutor;
+import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.ITableDeviceSchemaValidation;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
+import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.commons.io.FileUtils;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.TsFileSequenceReaderTimeseriesMetadataIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+
+import static org.apache.iotdb.db.queryengine.plan.execution.config.TableConfigTaskVisitor.validateDatabaseName;
+import static org.apache.iotdb.db.utils.constant.SqlConstant.ROOT;
+import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR_CHAR;
+
+public class LoadTsFileToTableModelAnalyzer extends LoadTsFileAnalyzer {
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(LoadTsFileToTableModelAnalyzer.class);
+
+ private final Metadata metadata;
+
+ public LoadTsFileToTableModelAnalyzer(
+ LoadTsFileStatement loadTsFileStatement, Metadata metadata, MPPQueryContext context) {
+ super(loadTsFileStatement, context);
+ this.metadata = metadata;
+ }
+
+ public LoadTsFileToTableModelAnalyzer(
+ LoadTsFile loadTsFileTableStatement, Metadata metadata, MPPQueryContext context) {
+ super(loadTsFileTableStatement, context);
+ this.metadata = metadata;
+ }
+
+ @Override
+ public IAnalysis analyzeFileByFile(IAnalysis analysis) {
+ // check if the system is read only
+ if (CommonDescriptor.getInstance().getConfig().isReadOnly()) {
+ analysis.setFinishQueryAfterAnalyze(true);
+ analysis.setFailStatus(
+ RpcUtils.getStatus(TSStatusCode.SYSTEM_READ_ONLY, LoadReadOnlyException.MESSAGE));
+ return analysis;
+ }
+
+ try {
+ autoCreateDatabase(database);
+ } catch (VerifyMetadataException e) {
+ LOGGER.warn("Auto create database failed: {}", database, e);
+ analysis.setFinishQueryAfterAnalyze(true);
+ analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, e.getMessage()));
+ return analysis;
+ }
+
+ if (!doAnalyzeFileByFile(analysis)) {
+ // return false means the analysis is failed because of exception
+ return analysis;
+ }
+
+ LOGGER.info("Load - Analysis Stage: all tsfiles have been analyzed.");
+
+ // data partition will be queried in the scheduler
+ setRealStatement(analysis);
+ return analysis;
+ }
+
+ @Override
+ protected void analyzeSingleTsFile(final File tsFile) throws IOException, AuthException {
+ try (final TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) {
+ // can be reused when constructing tsfile resource
+ final TsFileSequenceReaderTimeseriesMetadataIterator timeseriesMetadataIterator =
+ new TsFileSequenceReaderTimeseriesMetadataIterator(reader, true, 1);
+
+ // check if the tsfile is empty
+ if (!timeseriesMetadataIterator.hasNext()) {
+ throw new LoadEmptyFileException(tsFile.getAbsolutePath());
+ }
+
+ // check whether the tsfile is table-model or not
+ if (Objects.isNull(reader.readFileMetadata().getTableSchemaMap())
+ || reader.readFileMetadata().getTableSchemaMap().size() == 0) {
+ throw new SemanticException("Attempted to load a tree-model TsFile into table-model.");
+ }
+
+ // construct tsfile resource
+ final TsFileResource tsFileResource = constructTsFileResource(reader, tsFile);
+
+ for (Map.Entry<String, TableSchema> name2Schema :
+ reader.readFileMetadata().getTableSchemaMap().entrySet()) {
+ org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema realSchema =
+ metadata
+ .validateTableHeaderSchema(
+ database,
+ org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema
+ .fromTsFileTableSchema(name2Schema.getKey(), name2Schema.getValue()),
+ context,
+ true)
+ .orElse(null);
+ if (Objects.isNull(realSchema)) {
+ LOGGER.warn("Failed to validata schema for table {}", name2Schema.getValue());
+ }
+ }
+
+ long writePointCount = 0;
+
+ while (timeseriesMetadataIterator.hasNext()) {
+ final Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadata =
+ timeseriesMetadataIterator.next();
+
+ for (IDeviceID deviceId : device2TimeseriesMetadata.keySet()) {
+ final ITableDeviceSchemaValidation tableSchemaValidation =
+ createTableSchemaValidation(deviceId, this);
+ // TODO: currently, we only validate one device at a time for table model,
+ // may need to validate in batched manner in the future.
+ metadata.validateDeviceSchema(tableSchemaValidation, context);
+ }
+
+ if (!tsFileResource.resourceFileExists()) {
+ TsFileResourceUtils.updateTsFileResource(device2TimeseriesMetadata, tsFileResource);
+ }
+
+ writePointCount += getWritePointCount(device2TimeseriesMetadata);
+ }
+
+ TimestampPrecisionUtils.checkTimestampPrecision(tsFileResource.getFileEndTime());
+ tsFileResource.setStatus(TsFileResourceStatus.NORMAL);
+
+ addTsFileResource(tsFileResource);
+ addWritePointCount(writePointCount);
+ } catch (final LoadEmptyFileException loadEmptyFileException) {
+ LOGGER.warn("Failed to load empty file: {}", tsFile.getAbsolutePath());
+ if (isDeleteAfterLoad) {
+ FileUtils.deleteQuietly(tsFile);
+ }
+ }
+ }
+
+ private void autoCreateDatabase(final String database) throws VerifyMetadataException {
+ validateDatabaseName(database);
+ final CreateDBTask task =
+ new CreateDBTask(new TDatabaseSchema(ROOT + PATH_SEPARATOR_CHAR + database), true);
+ try {
+ final ListenableFuture<ConfigTaskResult> future =
+ task.execute(ClusterConfigTaskExecutor.getInstance());
+ final ConfigTaskResult result = future.get();
+ if (result.getStatusCode().getStatusCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new VerifyMetadataException(
+ String.format(
+ "Auto create database failed: %s, status code: %s",
+ database, result.getStatusCode()));
+ }
+ } catch (final ExecutionException | InterruptedException e) {
+ throw new VerifyMetadataException("Auto create database failed because: " + e.getMessage());
+ }
+ }
+
+ private ITableDeviceSchemaValidation createTableSchemaValidation(
+ IDeviceID deviceId, LoadTsFileToTableModelAnalyzer analyzer) {
+ return new ITableDeviceSchemaValidation() {
+
+ @Override
+ public String getDatabase() {
+ return analyzer.database;
+ }
+
+ @Override
+ public String getTableName() {
+ return deviceId.getTableName();
+ }
+
+ @Override
+ public List<Object[]> getDeviceIdList() {
+ return Collections.singletonList(
+ Arrays.copyOfRange(deviceId.getSegments(), 1, deviceId.getSegments().length));
+ }
+
+ @Override
+ public List<String> getAttributeColumnNameList() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<Object[]> getAttributeValueList() {
+ return Collections.singletonList(new Object[0]);
+ }
+ };
+ }
+
+ @Override
+ public void close() throws Exception {}
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java
new file mode 100644
index 00000000000..5552109999d
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileToTreeModelAnalyzer.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.analyze.load;
+
+import org.apache.iotdb.commons.auth.AuthException;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.LoadEmptyFileException;
+import org.apache.iotdb.db.exception.LoadReadOnlyException;
+import org.apache.iotdb.db.exception.sql.SemanticException;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
+import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.read.TsFileSequenceReaderTimeseriesMetadataIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class LoadTsFileToTreeModelAnalyzer extends LoadTsFileAnalyzer {
+ private static final Logger LOGGER = LoggerFactory.getLogger(LoadTsFileToTreeModelAnalyzer.class);
+
+ private final SchemaAutoCreatorAndVerifier schemaAutoCreatorAndVerifier;
+
+ public LoadTsFileToTreeModelAnalyzer(
+ LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) {
+ super(loadTsFileStatement, context);
+ this.schemaAutoCreatorAndVerifier = new SchemaAutoCreatorAndVerifier(this);
+ }
+
+ public LoadTsFileToTreeModelAnalyzer(
+ LoadTsFile loadTsFileTableStatement, MPPQueryContext context) {
+ super(loadTsFileTableStatement, context);
+ this.schemaAutoCreatorAndVerifier = new SchemaAutoCreatorAndVerifier(this);
+ }
+
+ @Override
+ public IAnalysis analyzeFileByFile(IAnalysis analysis) {
+ // check if the system is read only
+ if (CommonDescriptor.getInstance().getConfig().isReadOnly()) {
+ analysis.setFinishQueryAfterAnalyze(true);
+ analysis.setFailStatus(
+ RpcUtils.getStatus(TSStatusCode.SYSTEM_READ_ONLY, LoadReadOnlyException.MESSAGE));
+ return analysis;
+ }
+
+ if (!doAnalyzeFileByFile(analysis)) {
+ // return false means the analysis is failed because of exception
+ return analysis;
+ }
+
+ try {
+ schemaAutoCreatorAndVerifier.flush();
+ } catch (AuthException e) {
+ setFailAnalysisForAuthException(analysis, e);
+ return analysis;
+ } catch (Exception e) {
+ final String exceptionMessage =
+ String.format(
+ "Auto create or verify schema error when executing statement %s. Detail: %s.",
+ getStatementString(),
+ e.getMessage() == null ? e.getClass().getName() : e.getMessage());
+ LOGGER.warn(exceptionMessage, e);
+ analysis.setFinishQueryAfterAnalyze(true);
+ analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exceptionMessage));
+ return analysis;
+ }
+
+ LOGGER.info("Load - Analysis Stage: all tsfiles have been analyzed.");
+
+ // data partition will be queried in the scheduler
+ setRealStatement(analysis);
+ return analysis;
+ }
+
+ @Override
+ protected void analyzeSingleTsFile(final File tsFile) throws IOException, AuthException {
+ try (final TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) {
+ // can be reused when constructing tsfile resource
+ final TsFileSequenceReaderTimeseriesMetadataIterator timeseriesMetadataIterator =
+ new TsFileSequenceReaderTimeseriesMetadataIterator(reader, true, 1);
+
+ // check if the tsfile is empty
+ if (!timeseriesMetadataIterator.hasNext()) {
+ throw new LoadEmptyFileException(tsFile.getAbsolutePath());
+ }
+
+ // check whether the tsfile is tree-model or not
+ if (Objects.nonNull(reader.readFileMetadata().getTableSchemaMap())
+ && reader.readFileMetadata().getTableSchemaMap().size() != 0) {
+ throw new SemanticException("Attempted to load a table-model TsFile into tree-model.");
+ }
+
+ // construct tsfile resource
+ final TsFileResource tsFileResource = constructTsFileResource(reader, tsFile);
+
+ schemaAutoCreatorAndVerifier.setCurrentModificationsAndTimeIndex(tsFileResource, reader);
+
+ long writePointCount = 0;
+
+ final boolean isAutoCreateSchemaOrVerifySchemaEnabled =
+ IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled() || isVerifySchema();
+ while (timeseriesMetadataIterator.hasNext()) {
+ final Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadata =
+ timeseriesMetadataIterator.next();
+
+ if (isAutoCreateSchemaOrVerifySchemaEnabled) {
+ schemaAutoCreatorAndVerifier.autoCreateAndVerify(reader, device2TimeseriesMetadata);
+ }
+
+ if (!tsFileResource.resourceFileExists()) {
+ TsFileResourceUtils.updateTsFileResource(device2TimeseriesMetadata, tsFileResource);
+ }
+
+ // TODO: how to get the correct write point count when
+ // !isAutoCreateSchemaOrVerifySchemaEnabled
+ writePointCount += getWritePointCount(device2TimeseriesMetadata);
+ }
+ if (isAutoCreateSchemaOrVerifySchemaEnabled) {
+ schemaAutoCreatorAndVerifier.flushAndClearDeviceIsAlignedCacheIfNecessary();
+ }
+
+ TimestampPrecisionUtils.checkTimestampPrecision(tsFileResource.getFileEndTime());
+ tsFileResource.setStatus(TsFileResourceStatus.NORMAL);
+
+ addTsFileResource(tsFileResource);
+ addWritePointCount(writePointCount);
+ } catch (final LoadEmptyFileException loadEmptyFileException) {
+ LOGGER.warn("Failed to load empty file: {}", tsFile.getAbsolutePath());
+ if (isDeleteAfterLoad) {
+ FileUtils.deleteQuietly(tsFile);
+ }
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ schemaAutoCreatorAndVerifier.close();
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/SchemaAutoCreatorAndVerifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/SchemaAutoCreatorAndVerifier.java
new file mode 100644
index 00000000000..37e07a4ab17
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/SchemaAutoCreatorAndVerifier.java
@@ -0,0 +1,486 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.analyze.load;
+
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.auth.AuthException;
+import org.apache.iotdb.commons.auth.entity.PrivilegeType;
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.exception.ClientManagerException;
+import org.apache.iotdb.commons.consensus.ConfigRegionId;
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.MeasurementPath;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.schema.SchemaConstant;
+import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics;
+import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq;
+import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp;
+import org.apache.iotdb.db.auth.AuthorityChecker;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.LoadFileException;
+import org.apache.iotdb.db.exception.LoadRuntimeOutOfMemoryException;
+import org.apache.iotdb.db.exception.VerifyMetadataException;
+import org.apache.iotdb.db.exception.sql.SemanticException;
+import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
+import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
+import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
+import org.apache.iotdb.db.protocol.session.SessionManager;
+import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo;
+import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree;
+import org.apache.iotdb.db.queryengine.plan.Coordinator;
+import org.apache.iotdb.db.queryengine.plan.analyze.schema.SchemaValidator;
+import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult;
+import org.apache.iotdb.db.queryengine.plan.statement.Statement;
+import org.apache.iotdb.db.queryengine.plan.statement.metadata.DatabaseSchemaStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDatabaseStatement;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.utils.constant.SqlConstant;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.thrift.TException;
+import org.apache.tsfile.common.constant.TsFileConstant;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.TsFileSequenceReader;
+import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class SchemaAutoCreatorAndVerifier {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SchemaAutoCreatorAndVerifier.class);
+
+ private static final IClientManager<ConfigRegionId, ConfigNodeClient> CONFIG_NODE_CLIENT_MANAGER =
+ ConfigNodeClientManager.getInstance();
+
+ private final LoadTsFileAnalyzer loadTsFileAnalyzer;
+ private final LoadTsFileAnalyzeSchemaCache schemaCache;
+
+ SchemaAutoCreatorAndVerifier(LoadTsFileAnalyzer loadTsFileAnalyzer)
+ throws LoadRuntimeOutOfMemoryException {
+ this.loadTsFileAnalyzer = loadTsFileAnalyzer;
+ this.schemaCache = new LoadTsFileAnalyzeSchemaCache();
+ }
+
+ public void setCurrentModificationsAndTimeIndex(
+ TsFileResource resource, TsFileSequenceReader reader) throws IOException {
+ schemaCache.setCurrentModificationsAndTimeIndex(resource, reader);
+ }
+
+ public void autoCreateAndVerify(
+ TsFileSequenceReader reader,
+ Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadataList)
+ throws IOException, AuthException {
+ for (final Map.Entry<IDeviceID, List<TimeseriesMetadata>> entry :
+ device2TimeseriesMetadataList.entrySet()) {
+ final IDeviceID device = entry.getKey();
+
+ try {
+ if (schemaCache.isDeviceDeletedByMods(device)) {
+ continue;
+ }
+ } catch (IllegalPathException e) {
+ LOGGER.warn(
+ "Failed to check if device {} is deleted by mods. Will see it as not deleted.",
+ device,
+ e);
+ }
+
+ for (final TimeseriesMetadata timeseriesMetadata : entry.getValue()) {
+ try {
+ if (schemaCache.isTimeseriesDeletedByMods(device, timeseriesMetadata)) {
+ continue;
+ }
+ } catch (IllegalPathException e) {
+ LOGGER.warn(
+ "Failed to check if device {}, timeseries {} is deleted by mods. Will see it as not deleted.",
+ device,
+ timeseriesMetadata.getMeasurementId(),
+ e);
+ }
+
+ final TSDataType dataType = timeseriesMetadata.getTsDataType();
+ if (TSDataType.VECTOR.equals(dataType)) {
+ schemaCache
+ .clearDeviceIsAlignedCacheIfNecessary(); // must execute before add aligned cache
+ schemaCache.addIsAlignedCache(device, true, false);
+
+ // not a timeseries, skip
+ } else {
+ // check WRITE_DATA permission of timeseries
+ long startTime = System.nanoTime();
+ try {
+ String userName = loadTsFileAnalyzer.context.getSession().getUserName();
+ if (!AuthorityChecker.SUPER_USER.equals(userName)) {
+ TSStatus status;
+ try {
+ List<PartialPath> paths =
+ Collections.singletonList(
+ new MeasurementPath(device, timeseriesMetadata.getMeasurementId()));
+ status =
+ AuthorityChecker.getTSStatus(
+ AuthorityChecker.checkFullPathListPermission(
+ userName, paths, PrivilegeType.WRITE_DATA.ordinal()),
+ paths,
+ PrivilegeType.WRITE_DATA);
+ } catch (IllegalPathException e) {
+ throw new RuntimeException(e);
+ }
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new AuthException(
+ TSStatusCode.representOf(status.getCode()), status.getMessage());
+ }
+ }
+ } finally {
+ PerformanceOverviewMetrics.getInstance().recordAuthCost(System.nanoTime() - startTime);
+ }
+ final Pair<CompressionType, TSEncoding> compressionEncodingPair =
+ reader.readTimeseriesCompressionTypeAndEncoding(timeseriesMetadata);
+ schemaCache.addTimeSeries(
+ device,
+ new MeasurementSchema(
+ timeseriesMetadata.getMeasurementId(),
+ dataType,
+ compressionEncodingPair.getRight(),
+ compressionEncodingPair.getLeft()));
+
+ schemaCache.addIsAlignedCache(device, false, true);
+ if (!schemaCache.getDeviceIsAligned(device)) {
+ schemaCache.clearDeviceIsAlignedCacheIfNecessary();
+ }
+ }
+
+ if (schemaCache.shouldFlushTimeSeries()) {
+ flush();
+ }
+ }
+ }
+ }
+
+ /**
+ * This can only be invoked after all timeseries in the current tsfile have been processed.
+ * Otherwise, the isAligned status may be wrong.
+ */
+ public void flushAndClearDeviceIsAlignedCacheIfNecessary() throws SemanticException {
+ // avoid OOM when loading a tsfile with too many timeseries
+ // or loading too many tsfiles at the same time
+ schemaCache.clearDeviceIsAlignedCacheIfNecessary();
+ }
+
+ public void flush() throws AuthException {
+ doAutoCreateAndVerify();
+
+ schemaCache.clearTimeSeries();
+ }
+
+ private void doAutoCreateAndVerify() throws SemanticException, AuthException {
+ if (schemaCache.getDevice2TimeSeries().isEmpty()) {
+ return;
+ }
+
+ try {
+ if (loadTsFileAnalyzer.isVerifySchema()) {
+ makeSureNoDuplicatedMeasurementsInDevices();
+ }
+
+ if (loadTsFileAnalyzer.isAutoCreateDatabase()) {
+ autoCreateDatabase();
+ }
+
+ // schema fetcher will not auto create if config set
+ // isAutoCreateSchemaEnabled is false.
+ final ISchemaTree schemaTree = autoCreateSchema();
+
+ if (loadTsFileAnalyzer.isVerifySchema()) {
+ verifySchema(schemaTree);
+ }
+ } catch (AuthException e) {
+ throw e;
+ } catch (Exception e) {
+ LOGGER.warn("Auto create or verify schema error.", e);
+ throw new SemanticException(
+ String.format(
+ "Auto create or verify schema error when executing statement %s. Detail: %s.",
+ loadTsFileAnalyzer.getStatementString(), e.getMessage()));
+ }
+ }
+
+ private void makeSureNoDuplicatedMeasurementsInDevices() throws VerifyMetadataException {
+ for (final Map.Entry<IDeviceID, Set<MeasurementSchema>> entry :
+ schemaCache.getDevice2TimeSeries().entrySet()) {
+ final IDeviceID device = entry.getKey();
+ final Map<String, MeasurementSchema> measurement2Schema = new HashMap<>();
+ for (final MeasurementSchema timeseriesSchema : entry.getValue()) {
+ final String measurement = timeseriesSchema.getMeasurementId();
+ if (measurement2Schema.containsKey(measurement)) {
+ throw new VerifyMetadataException(
+ String.format("Duplicated measurements %s in device %s.", measurement, device));
+ }
+ measurement2Schema.put(measurement, timeseriesSchema);
+ }
+ }
+ }
+
+ private void autoCreateDatabase()
+ throws VerifyMetadataException, LoadFileException, IllegalPathException, AuthException {
+ final int databasePrefixNodesLength = loadTsFileAnalyzer.getDatabaseLevel() + 1;
+ final Set<PartialPath> databasesNeededToBeSet = new HashSet<>();
+
+ for (final IDeviceID device : schemaCache.getDevice2TimeSeries().keySet()) {
+ final PartialPath devicePath = new PartialPath(device);
+
+ final String[] devicePrefixNodes = devicePath.getNodes();
+ if (devicePrefixNodes.length < databasePrefixNodesLength) {
+ throw new VerifyMetadataException(
+ String.format(
+ "Database level %d is longer than device %s.", databasePrefixNodesLength, device));
+ }
+
+ final String[] databasePrefixNodes = new String[databasePrefixNodesLength];
+ System.arraycopy(devicePrefixNodes, 0, databasePrefixNodes, 0, databasePrefixNodesLength);
+
+ databasesNeededToBeSet.add(new PartialPath(databasePrefixNodes));
+ }
+
+ // 1. filter out the databases that already exist
+ if (schemaCache.getAlreadySetDatabases().isEmpty()) {
+ try (final ConfigNodeClient configNodeClient =
+ CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
+ final TGetDatabaseReq req =
+ new TGetDatabaseReq(
+ Arrays.asList(
+ new ShowDatabaseStatement(new PartialPath(SqlConstant.getSingleRootArray()))
+ .getPathPattern()
+ .getNodes()),
+ SchemaConstant.ALL_MATCH_SCOPE.serialize());
+ final TShowDatabaseResp resp = configNodeClient.showDatabase(req);
+
+ for (final String databaseName : resp.getDatabaseInfoMap().keySet()) {
+ schemaCache.addAlreadySetDatabase(new PartialPath(databaseName));
+ }
+ } catch (IOException | TException | ClientManagerException e) {
+ throw new LoadFileException(e);
+ }
+ }
+ databasesNeededToBeSet.removeAll(schemaCache.getAlreadySetDatabases());
+
+ // 2. create the databases that do not exist
+ for (final PartialPath databasePath : databasesNeededToBeSet) {
+ final DatabaseSchemaStatement statement =
+ new DatabaseSchemaStatement(DatabaseSchemaStatement.DatabaseSchemaStatementType.CREATE);
+ statement.setDatabasePath(databasePath);
+ // do not print exception log because it is not an error
+ statement.setEnablePrintExceptionLog(false);
+ executeSetDatabaseStatement(statement);
+
+ schemaCache.addAlreadySetDatabase(databasePath);
+ }
+ }
+
+ private void executeSetDatabaseStatement(Statement statement)
+ throws LoadFileException, AuthException {
+ // 1.check Authority
+ TSStatus status =
+ AuthorityChecker.checkAuthority(
+ statement, loadTsFileAnalyzer.context.getSession().getUserName());
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new AuthException(TSStatusCode.representOf(status.getCode()), status.getMessage());
+ }
+
+ // 2.execute setDatabase statement
+ final long queryId = SessionManager.getInstance().requestQueryId();
+ final ExecutionResult result =
+ Coordinator.getInstance()
+ .executeForTreeModel(
+ statement,
+ queryId,
+ null,
+ "",
+ loadTsFileAnalyzer.partitionFetcher,
+ loadTsFileAnalyzer.schemaFetcher,
+ IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold());
+ if (result.status.code != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ && result.status.code != TSStatusCode.DATABASE_ALREADY_EXISTS.getStatusCode()
+ // In tree model, if the user creates a conflict database concurrently, for instance, the
+ // database created by user is root.db.ss.a, the auto-creation failed database is root.db,
+ // we wait till "getOrCreatePartition" to judge if the time series (like root.db.ss.a.e /
+ // root.db.ss.a) conflicts with the created database. just do not throw exception here.
+ && result.status.code != TSStatusCode.DATABASE_CONFLICT.getStatusCode()) {
+ LOGGER.warn(
+ "Create database error, statement: {}, result status is: {}", statement, result.status);
+ throw new LoadFileException(
+ String.format(
+ "Create database error, statement: %s, result status is: %s",
+ statement, result.status));
+ }
+ }
+
+ private ISchemaTree autoCreateSchema() throws IllegalPathException {
+ final List<PartialPath> deviceList = new ArrayList<>();
+ final List<String[]> measurementList = new ArrayList<>();
+ final List<TSDataType[]> dataTypeList = new ArrayList<>();
+ final List<TSEncoding[]> encodingsList = new ArrayList<>();
+ final List<CompressionType[]> compressionTypesList = new ArrayList<>();
+ final List<Boolean> isAlignedList = new ArrayList<>();
+
+ for (final Map.Entry<IDeviceID, Set<MeasurementSchema>> entry :
+ schemaCache.getDevice2TimeSeries().entrySet()) {
+ final int measurementSize = entry.getValue().size();
+ final String[] measurements = new String[measurementSize];
+ final TSDataType[] tsDataTypes = new TSDataType[measurementSize];
+ final TSEncoding[] encodings = new TSEncoding[measurementSize];
+ final CompressionType[] compressionTypes = new CompressionType[measurementSize];
+
+ int index = 0;
+ for (final MeasurementSchema measurementSchema : entry.getValue()) {
+ measurements[index] = measurementSchema.getMeasurementId();
+ tsDataTypes[index] = measurementSchema.getType();
+ encodings[index] = measurementSchema.getEncodingType();
+ compressionTypes[index++] = measurementSchema.getCompressor();
+ }
+
+ deviceList.add(new PartialPath(entry.getKey()));
+ measurementList.add(measurements);
+ dataTypeList.add(tsDataTypes);
+ encodingsList.add(encodings);
+ compressionTypesList.add(compressionTypes);
+ isAlignedList.add(schemaCache.getDeviceIsAligned(entry.getKey()));
+ }
+
+ return SchemaValidator.validate(
+ loadTsFileAnalyzer.schemaFetcher,
+ deviceList,
+ measurementList,
+ dataTypeList,
+ encodingsList,
+ compressionTypesList,
+ isAlignedList,
+ loadTsFileAnalyzer.context);
+ }
+
+ private void verifySchema(ISchemaTree schemaTree)
+ throws VerifyMetadataException, IllegalPathException {
+ for (final Map.Entry<IDeviceID, Set<MeasurementSchema>> entry :
+ schemaCache.getDevice2TimeSeries().entrySet()) {
+ final IDeviceID device = entry.getKey();
+ final List<IMeasurementSchema> tsfileTimeseriesSchemas = new ArrayList<>(entry.getValue());
+ final DeviceSchemaInfo iotdbDeviceSchemaInfo =
+ schemaTree.searchDeviceSchemaInfo(
+ new PartialPath(device),
+ tsfileTimeseriesSchemas.stream()
+ .map(IMeasurementSchema::getMeasurementId)
+ .collect(Collectors.toList()));
+
+ if (iotdbDeviceSchemaInfo == null) {
+ throw new VerifyMetadataException(
+ String.format(
+ "Device %s does not exist in IoTDB and can not be created. "
+ + "Please check weather auto-create-schema is enabled.",
+ device));
+ }
+
+ // check device schema: is aligned or not
+ final boolean isAlignedInTsFile = schemaCache.getDeviceIsAligned(device);
+ final boolean isAlignedInIoTDB = iotdbDeviceSchemaInfo.isAligned();
+ if (isAlignedInTsFile != isAlignedInIoTDB) {
+ throw new VerifyMetadataException(
+ String.format(
+ "Device %s in TsFile is %s, but in IoTDB is %s.",
+ device,
+ isAlignedInTsFile ? "aligned" : "not aligned",
+ isAlignedInIoTDB ? "aligned" : "not aligned"));
+ }
+
+ // check timeseries schema
+ final List<IMeasurementSchema> iotdbTimeseriesSchemas =
+ iotdbDeviceSchemaInfo.getMeasurementSchemaList();
+ for (int i = 0, n = iotdbTimeseriesSchemas.size(); i < n; i++) {
+ final IMeasurementSchema tsFileSchema = tsfileTimeseriesSchemas.get(i);
+ final IMeasurementSchema iotdbSchema = iotdbTimeseriesSchemas.get(i);
+ if (iotdbSchema == null) {
+ throw new VerifyMetadataException(
+ String.format(
+ "Measurement %s does not exist in IoTDB and can not be created. "
+ + "Please check weather auto-create-schema is enabled.",
+ device + TsFileConstant.PATH_SEPARATOR + tsfileTimeseriesSchemas.get(i)));
+ }
+
+ // check datatype
+ if (!tsFileSchema.getType().equals(iotdbSchema.getType())) {
+ throw new VerifyMetadataException(
+ String.format(
+ "Measurement %s%s%s datatype not match, TsFile: %s, IoTDB: %s",
+ device,
+ TsFileConstant.PATH_SEPARATOR,
+ iotdbSchema.getMeasurementId(),
+ tsFileSchema.getType(),
+ iotdbSchema.getType()));
+ }
+
+ // check encoding
+ if (LOGGER.isDebugEnabled()
+ && !tsFileSchema.getEncodingType().equals(iotdbSchema.getEncodingType())) {
+ // we allow a measurement to have different encodings in different chunks
+ LOGGER.debug(
+ "Encoding type not match, measurement: {}{}{}, "
+ + "TsFile encoding: {}, IoTDB encoding: {}",
+ device,
+ TsFileConstant.PATH_SEPARATOR,
+ iotdbSchema.getMeasurementId(),
+ tsFileSchema.getEncodingType().name(),
+ iotdbSchema.getEncodingType().name());
+ }
+
+ // check compressor
+ if (LOGGER.isDebugEnabled()
+ && !tsFileSchema.getCompressor().equals(iotdbSchema.getCompressor())) {
+ // we allow a measurement to have different compressors in different chunks
+ LOGGER.debug(
+ "Compressor not match, measurement: {}{}{}, "
+ + "TsFile compressor: {}, IoTDB compressor: {}",
+ device,
+ TsFileConstant.PATH_SEPARATOR,
+ iotdbSchema.getMeasurementId(),
+ tsFileSchema.getCompressor().name(),
+ iotdbSchema.getCompressor().name());
+ }
+ }
+ }
+ }
+
+ public void close() {
+ schemaCache.close();
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
index 3a177923f3f..c26780185a3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
@@ -149,18 +149,8 @@ public class TableConfigTaskVisitor extends AstVisitor<IConfigTask, MPPQueryCont
final TDatabaseSchema schema = new TDatabaseSchema();
final String dbName = node.getDbName();
- // Check database length here
- // We need to calculate the database name without "root."
- if (dbName.contains(PATH_SEPARATOR)
- || !IoTDBConfig.STORAGE_GROUP_PATTERN.matcher(dbName).matches()
- || dbName.length() > MAX_DATABASE_NAME_LENGTH) {
- throw new SemanticException(
- new IllegalPathException(
- dbName,
- dbName.length() > MAX_DATABASE_NAME_LENGTH
- ? "the length of database name shall not exceed " + MAX_DATABASE_NAME_LENGTH
- : "the database name can only contain english or chinese characters, numbers, backticks and underscores."));
- }
+ validateDatabaseName(dbName);
+
schema.setName(ROOT + PATH_SEPARATOR_CHAR + dbName);
for (final Property property : node.getProperties()) {
@@ -330,6 +320,21 @@ public class TableConfigTaskVisitor extends AstVisitor<IConfigTask, MPPQueryCont
node.ifExists());
}
+ public static void validateDatabaseName(String dbName) throws SemanticException {
+ // Check database length here
+ // We need to calculate the database name without "root."
+ if (dbName.contains(PATH_SEPARATOR)
+ || !IoTDBConfig.STORAGE_GROUP_PATTERN.matcher(dbName).matches()
+ || dbName.length() > MAX_DATABASE_NAME_LENGTH) {
+ throw new SemanticException(
+ new IllegalPathException(
+ dbName,
+ dbName.length() > MAX_DATABASE_NAME_LENGTH
+ ? "the length of database name shall not exceed " + MAX_DATABASE_NAME_LENGTH
+ : "the database name can only contain english or chinese characters, numbers, backticks and underscores."));
+ }
+ }
+
public Pair<String, String> splitQualifiedName(final QualifiedName name) {
String database = clientSession.getDatabaseName();
if (name.getPrefix().isPresent()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java
index 828f0bd9611..fb6a9d6f041 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java
@@ -87,6 +87,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement
import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement;
import org.apache.iotdb.db.schemaengine.template.Template;
+import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -513,8 +514,16 @@ public class LogicalPlanVisitor extends StatementVisitor<PlanNode, MPPQueryConte
@Override
public PlanNode visitLoadFile(LoadTsFileStatement loadTsFileStatement, MPPQueryContext context) {
+ final List<Boolean> isTableModel = new ArrayList<>();
+ for (int i = 0; i < loadTsFileStatement.getResources().size(); i++) {
+ isTableModel.add(
+ loadTsFileStatement.getModel().equals(LoadTsFileConfigurator.MODEL_TABLE_VALUE));
+ }
return new LoadTsFileNode(
- context.getQueryId().genPlanNodeId(), loadTsFileStatement.getResources());
+ context.getQueryId().genPlanNodeId(),
+ loadTsFileStatement.getResources(),
+ isTableModel,
+ loadTsFileStatement.getDatabase());
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java
index 2232548a124..d58ce353b87 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java
@@ -56,6 +56,8 @@ public class LoadSingleTsFileNode extends WritePlanNode {
private final File tsFile;
private final TsFileResource resource;
+ private final boolean isTableModel;
+ private final String database;
private final boolean deleteAfterLoad;
private final long writePointCount;
private boolean needDecodeTsFile;
@@ -63,10 +65,17 @@ public class LoadSingleTsFileNode extends WritePlanNode {
private TRegionReplicaSet localRegionReplicaSet;
public LoadSingleTsFileNode(
- PlanNodeId id, TsFileResource resource, boolean deleteAfterLoad, long writePointCount) {
+ PlanNodeId id,
+ TsFileResource resource,
+ boolean isTableModel,
+ String database,
+ boolean deleteAfterLoad,
+ long writePointCount) {
super(id);
this.tsFile = resource.getTsFile();
this.resource = resource;
+ this.isTableModel = isTableModel;
+ this.database = database;
this.deleteAfterLoad = deleteAfterLoad;
this.writePointCount = writePointCount;
}
@@ -130,6 +139,10 @@ public class LoadSingleTsFileNode extends WritePlanNode {
return deleteAfterLoad;
}
+ public boolean isTableModel() {
+ return isTableModel;
+ }
+
public long getWritePointCount() {
return writePointCount;
}
@@ -147,6 +160,10 @@ public class LoadSingleTsFileNode extends WritePlanNode {
return resource;
}
+ public String getDatabase() {
+ return database;
+ }
+
@Override
public TRegionReplicaSet getRegionReplicaSet() {
return null;
@@ -227,6 +244,8 @@ public class LoadSingleTsFileNode extends WritePlanNode {
LoadSingleTsFileNode loadSingleTsFileNode = (LoadSingleTsFileNode) o;
return Objects.equals(tsFile, loadSingleTsFileNode.tsFile)
&& Objects.equals(resource, loadSingleTsFileNode.resource)
+ && Objects.equals(isTableModel, loadSingleTsFileNode.isTableModel)
+ && Objects.equals(database, loadSingleTsFileNode.database)
&& Objects.equals(needDecodeTsFile, loadSingleTsFileNode.needDecodeTsFile)
&& Objects.equals(deleteAfterLoad, loadSingleTsFileNode.deleteAfterLoad)
&& Objects.equals(localRegionReplicaSet, loadSingleTsFileNode.localRegionReplicaSet);
@@ -234,6 +253,13 @@ public class LoadSingleTsFileNode extends WritePlanNode {
@Override
public int hashCode() {
- return Objects.hash(tsFile, resource, needDecodeTsFile, deleteAfterLoad, localRegionReplicaSet);
+ return Objects.hash(
+ tsFile,
+ resource,
+ isTableModel,
+ database,
+ needDecodeTsFile,
+ deleteAfterLoad,
+ localRegionReplicaSet);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java
index be5de4cc819..2dba20a0bd3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement;
import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -42,10 +43,15 @@ import java.util.Objects;
public class LoadTsFileNode extends WritePlanNode {
private final List<TsFileResource> resources;
+ private final List<Boolean> isTableModel;
+ private final String database;
- public LoadTsFileNode(PlanNodeId id, List<TsFileResource> resources) {
+ public LoadTsFileNode(
+ PlanNodeId id, List<TsFileResource> resources, List<Boolean> isTableModel, String database) {
super(id);
this.resources = resources;
+ this.isTableModel = isTableModel;
+ this.database = database;
}
@Override
@@ -90,25 +96,54 @@ public class LoadTsFileNode extends WritePlanNode {
@Override
public List<WritePlanNode> splitByPartition(IAnalysis analysis) {
+ if (analysis instanceof Analysis) {
+ return splitByPartitionForTreeModel((Analysis) analysis);
+ } else {
+ return splitByPartitionForTableModel(
+ (org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis) analysis);
+ }
+ }
+
+ private List<WritePlanNode> splitByPartitionForTreeModel(Analysis analysis) {
List<WritePlanNode> res = new ArrayList<>();
LoadTsFileStatement statement =
- ((Analysis) analysis).getTreeStatement() instanceof PipeEnrichedStatement
+ analysis.getTreeStatement() instanceof PipeEnrichedStatement
? (LoadTsFileStatement)
- ((PipeEnrichedStatement) ((Analysis) analysis).getTreeStatement())
- .getInnerStatement()
- : (LoadTsFileStatement) ((Analysis) analysis).getTreeStatement();
+ ((PipeEnrichedStatement) analysis.getTreeStatement()).getInnerStatement()
+ : (LoadTsFileStatement) analysis.getTreeStatement();
for (int i = 0; i < resources.size(); i++) {
res.add(
new LoadSingleTsFileNode(
getPlanNodeId(),
resources.get(i),
+ isTableModel.get(i),
+ database,
statement.isDeleteAfterLoad(),
statement.getWritePointCount(i)));
}
return res;
}
+ private List<WritePlanNode> splitByPartitionForTableModel(
+ org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis analysis) {
+ List<WritePlanNode> res = new ArrayList<>();
+ LoadTsFile statement = (LoadTsFile) analysis.getStatement();
+ for (int i = 0; i < resources.size(); i++) {
+ if (statement != null) {
+ res.add(
+ new LoadSingleTsFileNode(
+ getPlanNodeId(),
+ resources.get(i),
+ isTableModel.get(i),
+ database,
+ statement.isDeleteAfterLoad(),
+ statement.getWritePointCount(i)));
+ }
+ }
+ return res;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -118,11 +153,13 @@ public class LoadTsFileNode extends WritePlanNode {
return false;
}
LoadTsFileNode loadTsFileNode = (LoadTsFileNode) o;
- return Objects.equals(resources, loadTsFileNode.resources);
+ return Objects.equals(resources, loadTsFileNode.resources)
+ && Objects.equals(database, loadTsFileNode.database)
+ && Objects.equals(isTableModel, loadTsFileNode.isTableModel);
}
@Override
public int hashCode() {
- return Objects.hash(resources);
+ return Objects.hash(resources, database, isTableModel);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
index 16bdd6c1e75..4e3ecf0092d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
@@ -29,6 +29,9 @@ import org.apache.iotdb.db.queryengine.execution.warnings.IoTDBWarning;
import org.apache.iotdb.db.queryengine.execution.warnings.WarningCollector;
import org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeUtils;
import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
+import org.apache.iotdb.db.queryengine.plan.analyze.load.LoadTsFileAnalyzer;
+import org.apache.iotdb.db.queryengine.plan.analyze.load.LoadTsFileToTableModelAnalyzer;
+import org.apache.iotdb.db.queryengine.plan.analyze.load.LoadTsFileToTreeModelAnalyzer;
import org.apache.iotdb.db.queryengine.plan.analyze.schema.SchemaValidator;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
@@ -88,6 +91,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinOn;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinUsing;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Limit;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NaturalJoin;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node;
@@ -133,6 +137,10 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStat
import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement;
import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
+import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator;
+import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.rpc.TSStatusCode;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
@@ -188,6 +196,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join.Type.
import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join.Type.RIGHT;
import static org.apache.iotdb.db.queryengine.plan.relational.sql.util.AstUtil.preOrder;
import static org.apache.iotdb.db.queryengine.plan.relational.utils.NodeUtils.getSortItemsFromOrderBy;
+import static org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet.ANALYSIS;
import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN;
public class StatementAnalyzer {
@@ -496,6 +505,46 @@ public class StatementAnalyzer {
throw new SemanticException("Delete statement is not supported yet.");
}
+ @Override
+ protected Scope visitLoadTsFile(LoadTsFile node, Optional<Scope> scope) {
+ queryContext.setQueryType(QueryType.WRITE);
+
+ final long startTime = System.nanoTime();
+ try (final LoadTsFileAnalyzer loadTsFileAnalyzer = getAnalyzer(node)) {
+ loadTsFileAnalyzer.analyzeFileByFile(analysis);
+ } catch (final Exception e) {
+ final String exceptionMessage =
+ String.format(
+ "Failed to execute load tsfile statement %s. Detail: %s",
+ node, e.getMessage() == null ? e.getClass().getName() : e.getMessage());
+ analysis.setFinishQueryAfterAnalyze(true);
+ analysis.setFailStatus(RpcUtils.getStatus(TSStatusCode.LOAD_FILE_ERROR, exceptionMessage));
+ } finally {
+ LoadTsFileCostMetricsSet.getInstance()
+ .recordPhaseTimeCost(ANALYSIS, System.nanoTime() - startTime);
+ }
+
+ return createAndAssignScope(node, scope);
+ }
+
+ private LoadTsFileAnalyzer getAnalyzer(LoadTsFile loadTsFile) {
+ if (Objects.equals(loadTsFile.getModel(), LoadTsFileConfigurator.MODEL_TABLE_VALUE)) {
+ // Load to table-model
+ if (Objects.isNull(loadTsFile.getDatabase())) {
+ // If database is not specified, use the database from current session.
+ // If still not specified, throw an exception.
+ if (!queryContext.getDatabaseName().isPresent()) {
+ throw new SemanticException("Database is not specified");
+ }
+ loadTsFile.setDatabase(queryContext.getDatabaseName().get());
+ }
+ return new LoadTsFileToTableModelAnalyzer(loadTsFile, metadata, queryContext);
+ } else {
+ // Load to tree-model
+ return new LoadTsFileToTreeModelAnalyzer(loadTsFile, queryContext);
+ }
+ }
+
@Override
protected Scope visitExplain(Explain node, Optional<Scope> context) {
analysis.setFinishQueryAfterAnalyze();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableSchema.java
index e2ff268ef55..d9a21b0b6b0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableSchema.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableSchema.java
@@ -79,6 +79,22 @@ public class TableSchema {
tableName, measurementSchemas, columnTypes);
}
+ public static TableSchema fromTsFileTableSchema(
+ String tableName, org.apache.tsfile.file.metadata.TableSchema tsFileTableSchema) {
+ List<ColumnSchema> columns = new ArrayList<>();
+ for (int i = 0; i < tsFileTableSchema.getColumnSchemas().size(); i++) {
+ columns.add(
+ new ColumnSchema(
+ tsFileTableSchema.getColumnSchemas().get(i).getMeasurementId(),
+ InternalTypeManager.fromTSDataType(
+ tsFileTableSchema.getColumnSchemas().get(i).getType()),
+ false,
+ TsTableColumnCategory.fromTsFileColumnType(
+ tsFileTableSchema.getColumnTypes().get(i))));
+ }
+ return new TableSchema(tableName, columns);
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
index 9222aa1f102..b80ce36abbb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
@@ -111,7 +111,7 @@ public class TableDeviceSchemaFetcher {
SessionManager.getInstance().getCurrSession(),
queryId,
SessionManager.getInstance()
- .getSessionInfo(SessionManager.getInstance().getCurrSession()),
+ .getSessionInfoOfTableModel(SessionManager.getInstance().getCurrSession()),
"Fetch Device for insert",
LocalExecutionPlanner.getInstance().metadata,
config.getQueryTimeoutThreshold());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
index c587774bcef..b50cb013d8b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
@@ -18,6 +18,7 @@ import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.QueryId;
import org.apache.iotdb.db.queryengine.common.SessionInfo;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode;
@@ -48,6 +49,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InsertTablet;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Intersect;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinUsing;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName;
@@ -62,6 +64,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
+import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -692,4 +695,17 @@ public class RelationPlanner extends AstVisitor<RelationPlan, Void> {
return new RelationPlan(
relationalInsertRowsNode, analysis.getRootScope(), Collections.emptyList());
}
+
+ @Override
+ protected RelationPlan visitLoadTsFile(LoadTsFile node, Void context) {
+ final List<Boolean> isTableModel = new ArrayList<>();
+ for (int i = 0; i < node.getResources().size(); i++) {
+ isTableModel.add(node.getModel().equals(LoadTsFileConfigurator.MODEL_TABLE_VALUE));
+ }
+ return new RelationPlan(
+ new LoadTsFileNode(
+ idAllocator.genPlanNodeId(), node.getResources(), isTableModel, node.getDatabase()),
+ analysis.getRootScope(),
+ Collections.emptyList());
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java
index cb83d60007b..70d703afd6e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java
@@ -46,6 +46,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountDevice;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateOrUpdateDevice;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement;
@@ -190,6 +191,9 @@ public class TableLogicalPlanner {
if (statement instanceof WrappedStatement) {
return createRelationPlan(analysis, ((WrappedStatement) statement));
}
+ if (statement instanceof LoadTsFile) {
+ return createRelationPlan(analysis, (LoadTsFile) statement);
+ }
throw new IllegalStateException(
"Unsupported statement type: " + statement.getClass().getSimpleName());
}
@@ -235,6 +239,10 @@ public class TableLogicalPlanner {
return getRelationPlanner(analysis).process(statement, null);
}
+ private RelationPlan createRelationPlan(Analysis analysis, LoadTsFile loadTsFile) {
+ return getRelationPlanner(analysis).process(loadTsFile, null);
+ }
+
private RelationPlan createRelationPlan(Analysis analysis, Query query) {
return getRelationPlanner(analysis).process(query, null);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java
index 1b90384efc3..6ac7b588937 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.execution.QueryStateMachine;
import org.apache.iotdb.db.queryengine.execution.warnings.WarningCollector;
+import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher;
import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis;
import org.apache.iotdb.db.queryengine.plan.planner.IPlanner;
import org.apache.iotdb.db.queryengine.plan.planner.plan.DistributedQueryPlan;
@@ -37,11 +38,13 @@ import org.apache.iotdb.db.queryengine.plan.relational.analyzer.StatementAnalyze
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
import org.apache.iotdb.db.queryengine.plan.relational.planner.distribute.TableDistributedPlanner;
import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement;
import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser;
import org.apache.iotdb.db.queryengine.plan.scheduler.ClusterScheduler;
import org.apache.iotdb.db.queryengine.plan.scheduler.IScheduler;
+import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
import org.apache.iotdb.rpc.RpcUtils;
@@ -130,17 +133,29 @@ public class TableModelPlanner implements IPlanner {
DistributedQueryPlan distributedPlan,
MPPQueryContext context,
QueryStateMachine stateMachine) {
- IScheduler scheduler =
- new ClusterScheduler(
- context,
- stateMachine,
- distributedPlan.getInstances(),
- context.getQueryType(),
- executor,
- writeOperationExecutor,
- scheduledExecutor,
- syncInternalServiceClientManager,
- asyncInternalServiceClientManager);
+ IScheduler scheduler;
+ if (statement instanceof LoadTsFile) {
+ scheduler =
+ new LoadTsFileScheduler(
+ distributedPlan,
+ context,
+ stateMachine,
+ syncInternalServiceClientManager,
+ ClusterPartitionFetcher.getInstance(),
+ false);
+ } else {
+ scheduler =
+ new ClusterScheduler(
+ context,
+ stateMachine,
+ distributedPlan.getInstances(),
+ context.getQueryType(),
+ executor,
+ writeOperationExecutor,
+ scheduledExecutor,
+ syncInternalServiceClientManager,
+ asyncInternalServiceClientManager);
+ }
scheduler.start();
return scheduler;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java
index 76598ae1bce..103ffd90809 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java
@@ -542,4 +542,8 @@ public abstract class AstVisitor<R, C> {
protected R visitShowPipePlugins(ShowPipePlugins node, C context) {
return visitStatement(node, context);
}
+
+ protected R visitLoadTsFile(LoadTsFile node, C context) {
+ return visitStatement(node, context);
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java
new file mode 100644
index 00000000000..808c577b05b
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.sql.SemanticException;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+import static java.util.Objects.requireNonNull;
+
+public class LoadTsFile extends Statement {
+ private final String filePath;
+
+ private final File file;
+ private int databaseLevel; // For loading to tree-model only
+ private String database; // For loading to table-model only
+ private boolean deleteAfterLoad;
+ private boolean autoCreateDatabase;
+ private String model = LoadTsFileConfigurator.MODEL_TABLE_VALUE;
+
+ private final Map<String, String> loadAttributes;
+
+ private final List<File> tsFiles;
+ private final List<TsFileResource> resources;
+ private final List<Long> writePointCountList;
+
+ public LoadTsFile(NodeLocation location, String filePath, Map<String, String> loadAttributes) {
+ super(requireNonNull(location, "location is null"));
+ this.filePath = requireNonNull(filePath, "filePath is null");
+
+ this.file = new File(filePath);
+ this.databaseLevel = IoTDBDescriptor.getInstance().getConfig().getDefaultStorageGroupLevel();
+ this.deleteAfterLoad = false;
+ this.autoCreateDatabase = IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled();
+ this.resources = new ArrayList<>();
+ this.writePointCountList = new ArrayList<>();
+ this.loadAttributes = loadAttributes;
+ initAttributes();
+
+ try {
+ this.tsFiles =
+ org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement.processTsFile(
+ file);
+ } catch (FileNotFoundException e) {
+ throw new SemanticException(e);
+ }
+ }
+
+ public String getFilePath() {
+ return filePath;
+ }
+
+ public Map<String, String> getLoadAttributes() {
+ return loadAttributes;
+ }
+
+ public void setAutoCreateDatabase(boolean autoCreateDatabase) {
+ this.autoCreateDatabase = autoCreateDatabase;
+ }
+
+ public boolean isDeleteAfterLoad() {
+ return deleteAfterLoad;
+ }
+
+ public boolean isAutoCreateDatabase() {
+ return autoCreateDatabase;
+ }
+
+ public int getDatabaseLevel() {
+ return databaseLevel;
+ }
+
+ public String getDatabase() {
+ return database;
+ }
+
+ public void setDatabase(String database) {
+ this.database = database;
+ }
+
+ public String getModel() {
+ return model;
+ }
+
+ public List<File> getTsFiles() {
+ return tsFiles;
+ }
+
+ public void addTsFileResource(TsFileResource resource) {
+ resources.add(resource);
+ }
+
+ public List<TsFileResource> getResources() {
+ return resources;
+ }
+
+ public void addWritePointCount(long writePointCount) {
+ writePointCountList.add(writePointCount);
+ }
+
+ public long getWritePointCount(int resourceIndex) {
+ return writePointCountList.get(resourceIndex);
+ }
+
+ private void initAttributes() {
+ this.databaseLevel = LoadTsFileConfigurator.parseOrGetDefaultDatabaseLevel(loadAttributes);
+ this.database = LoadTsFileConfigurator.parseDatabaseName(loadAttributes);
+ this.deleteAfterLoad = LoadTsFileConfigurator.parseOrGetDefaultOnSuccess(loadAttributes);
+ this.model =
+ LoadTsFileConfigurator.parseOrGetDefaultModel(
+ loadAttributes, LoadTsFileConfigurator.MODEL_TABLE_VALUE);
+ }
+
+ @Override
+ public <R, C> R accept(AstVisitor<R, C> visitor, C context) {
+ return visitor.visitLoadTsFile(this, context);
+ }
+
+ @Override
+ public List<? extends Node> getChildren() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(filePath, loadAttributes);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if ((obj == null) || (getClass() != obj.getClass())) {
+ return false;
+ }
+ LoadTsFile other = (LoadTsFile) obj;
+ return Objects.equals(filePath, other.filePath)
+ && Objects.equals(loadAttributes, other.loadAttributes);
+ }
+
+ @Override
+ public String toString() {
+ return toStringHelper(this)
+ .add("filePath", filePath)
+ .add("loadAttributes", loadAttributes)
+ .toString();
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
index 54b7060fd50..ce4ae9f7a52 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
@@ -89,6 +89,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinUsing;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LikePredicate;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Limit;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NaturalJoin;
@@ -159,6 +160,7 @@ import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.Ta
import org.apache.iotdb.db.relational.grammar.sql.RelationalSqlBaseVisitor;
import org.apache.iotdb.db.relational.grammar.sql.RelationalSqlLexer;
import org.apache.iotdb.db.relational.grammar.sql.RelationalSqlParser;
+import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator;
import org.apache.iotdb.db.utils.DateTimeUtils;
import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
@@ -559,7 +561,40 @@ public class AstBuilder extends RelationalSqlBaseVisitor<Node> {
@Override
public Node visitLoadTsFileStatement(RelationalSqlParser.LoadTsFileStatementContext ctx) {
- return super.visitLoadTsFileStatement(ctx);
+ final Map<String, String> loadTsFileAttributes = new HashMap<>();
+ if (ctx.loadFileWithAttributeClauses() != null) {
+ for (RelationalSqlParser.LoadFileWithAttributeClauseContext attributeContext :
+ ctx.loadFileWithAttributeClauses().loadFileWithAttributeClause()) {
+ final String key =
+ parseStringLiteral(attributeContext.loadFileWithKey.getText()).trim().toLowerCase();
+ final String value =
+ parseStringLiteral(attributeContext.loadFileWithValue.getText()).trim().toLowerCase();
+
+ LoadTsFileConfigurator.validateParameters(key, value);
+ loadTsFileAttributes.put(key, value);
+ }
+ }
+
+ return new LoadTsFile(
+ getLocation(ctx), ((StringLiteral) visit(ctx.fileName)).getValue(), loadTsFileAttributes);
+ }
+
+ public static String parseStringLiteral(String src) {
+ if (2 <= src.length()) {
+ // do not unescape string
+ String unWrappedString = src.substring(1, src.length() - 1);
+ if (src.charAt(0) == '\"' && src.charAt(src.length() - 1) == '\"') {
+ // replace "" with "
+ String replaced = unWrappedString.replace("\"\"", "\"");
+ return replaced.length() == 0 ? "" : replaced;
+ }
+ if ((src.charAt(0) == '\'' && src.charAt(src.length() - 1) == '\'')) {
+ // replace '' with '
+ String replaced = unWrappedString.replace("''", "'");
+ return replaced.length() == 0 ? "" : replaced;
+ }
+ }
+ return src;
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java
index b03854d2747..355f56cdea1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java
@@ -50,6 +50,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinCriteria;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinOn;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.JoinUsing;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Limit;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NaturalJoin;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Offset;
@@ -797,6 +798,30 @@ public final class SqlFormatter {
return null;
}
+ @Override
+ protected Void visitLoadTsFile(final LoadTsFile node, final Integer indent) {
+ builder.append("LOAD ");
+ builder.append("\"" + node.getFilePath() + "\"");
+ builder.append(" \n");
+
+ builder
+ .append("WITH (")
+ .append("\n")
+ .append(
+ node.getLoadAttributes().entrySet().stream()
+ .map(
+ entry ->
+ indentString(1)
+ + "\""
+ + entry.getKey()
+ + "\" = \""
+ + entry.getValue()
+ + "\"")
+ .collect(joining(", " + "\n")))
+ .append(")\n");
+ return null;
+ }
+
@Override
protected Void visitCreatePipe(CreatePipe node, Integer context) {
builder.append("CREATE PIPE ");
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java
index 80c25e3df2d..6f2aaef76ab 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java
@@ -37,6 +37,7 @@ import static org.apache.tsfile.read.common.type.IntType.INT32;
import static org.apache.tsfile.read.common.type.LongType.INT64;
import static org.apache.tsfile.read.common.type.StringType.STRING;
import static org.apache.tsfile.read.common.type.TimestampType.TIMESTAMP;
+import static org.apache.tsfile.read.common.type.UnknownType.UNKNOWN;
public class InternalTypeManager implements TypeManager {
@@ -53,6 +54,7 @@ public class InternalTypeManager implements TypeManager {
types.put(new TypeSignature(TypeEnum.BLOB.name().toLowerCase(Locale.ENGLISH)), BLOB);
types.put(new TypeSignature(TypeEnum.DATE.name().toLowerCase(Locale.ENGLISH)), DATE);
types.put(new TypeSignature(TypeEnum.TIMESTAMP.name().toLowerCase(Locale.ENGLISH)), TIMESTAMP);
+ types.put(new TypeSignature(TypeEnum.UNKNOWN.name().toLowerCase(Locale.ENGLISH)), UNKNOWN);
}
@Override
@@ -107,4 +109,33 @@ public class InternalTypeManager implements TypeManager {
throw new IllegalArgumentException();
}
}
+
+ public static Type fromTSDataType(TSDataType dataType) {
+ switch (dataType) {
+ case TEXT:
+ return TEXT;
+ case FLOAT:
+ return FLOAT;
+ case DOUBLE:
+ return DOUBLE;
+ case INT32:
+ return INT32;
+ case INT64:
+ return INT64;
+ case BOOLEAN:
+ return BOOLEAN;
+ case UNKNOWN:
+ return UNKNOWN;
+ case DATE:
+ return DATE;
+ case TIMESTAMP:
+ return TIMESTAMP;
+ case BLOB:
+ return BLOB;
+ case STRING:
+ return STRING;
+ default:
+ throw new IllegalArgumentException();
+ }
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
index 91ac91f8714..fc116f409dc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
@@ -161,6 +161,12 @@ public class LoadTsFileScheduler implements IScheduler {
final LoadSingleTsFileNode node = tsFileNodeList.get(i);
final String filePath = node.getTsFileResource().getTsFilePath();
+ if (node.isTableModel()) {
+ partitionFetcher.setDatabase(node.getDatabase());
+ } else {
+ partitionFetcher.setDatabase(null);
+ }
+
boolean isLoadSingleTsFileSuccess = true;
boolean shouldRemoveFileFromLoadingSet = false;
try {
@@ -581,8 +587,7 @@ public class LoadTsFileScheduler implements IScheduler {
.map(
data ->
new Pair<>(
- (IDeviceID)
- IDeviceID.Factory.DEFAULT_FACTORY.create(data.getDevice()),
+ IDeviceID.Factory.DEFAULT_FACTORY.create(data.getDevice()),
data.getTimePartitionSlot()))
.collect(Collectors.toList()),
scheduler.queryContext.getSession().getUserName());
@@ -634,11 +639,16 @@ public class LoadTsFileScheduler implements IScheduler {
private static class DataPartitionBatchFetcher {
private final IPartitionFetcher fetcher;
+ private String database;
public DataPartitionBatchFetcher(IPartitionFetcher fetcher) {
this.fetcher = fetcher;
}
+ public void setDatabase(String database) {
+ this.database = database;
+ }
+
public List<TRegionReplicaSet> queryDataPartition(
List<Pair<IDeviceID, TTimePartitionSlot>> slotList, String userName) {
List<TRegionReplicaSet> replicaSets = new ArrayList<>();
@@ -651,7 +661,14 @@ public class LoadTsFileScheduler implements IScheduler {
fetcher.getOrCreateDataPartition(toQueryParam(subSlotList), userName);
replicaSets.addAll(
subSlotList.stream()
- .map(pair -> dataPartition.getDataRegionReplicaSetForWriting(pair.left, pair.right))
+ .map(
+ pair ->
+ // (database != null) means this file will be loaded into table-model
+ database != null
+ ? dataPartition.getDataRegionReplicaSetForWriting(
+ pair.left, pair.right, database)
+ : dataPartition.getDataRegionReplicaSetForWriting(
+ pair.left, pair.right))
.collect(Collectors.toList()));
}
return replicaSets;
@@ -666,8 +683,15 @@ public class LoadTsFileScheduler implements IScheduler {
.entrySet()
.stream()
.map(
- entry ->
- new DataPartitionQueryParam(entry.getKey(), new ArrayList<>(entry.getValue())))
+ entry -> {
+ DataPartitionQueryParam queryParam =
+ new DataPartitionQueryParam(entry.getKey(), new ArrayList<>(entry.getValue()));
+ // (database != null) means this file will be loaded into table-model
+ if (database != null) {
+ queryParam.setDatabaseName(database);
+ }
+ return queryParam;
+ })
.collect(Collectors.toList());
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
index 1fb3e5f1b92..3316c126e82 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
@@ -42,10 +42,12 @@ import java.util.Map;
public class LoadTsFileStatement extends Statement {
private final File file;
- private int databaseLevel;
+ private int databaseLevel; // For loading to tree-model only
+ private String database; // For loading to table-model only
private boolean verifySchema;
private boolean deleteAfterLoad;
private boolean autoCreateDatabase;
+ private String model = LoadTsFileConfigurator.MODEL_TREE_VALUE;
private Map<String, String> loadAttributes;
@@ -59,15 +61,15 @@ public class LoadTsFileStatement extends Statement {
this.verifySchema = true;
this.deleteAfterLoad = false;
this.autoCreateDatabase = IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled();
- this.tsFiles = new ArrayList<>();
this.resources = new ArrayList<>();
this.writePointCountList = new ArrayList<>();
this.statementType = StatementType.MULTI_BATCH_INSERT;
- processTsFile(filePath);
+ this.tsFiles = processTsFile(file);
}
- private void processTsFile(final String filePath) throws FileNotFoundException {
+ public static List<File> processTsFile(final File file) throws FileNotFoundException {
+ final List<File> tsFiles = new ArrayList<>();
if (file.isFile()) {
tsFiles.add(file);
} else {
@@ -75,11 +77,12 @@ public class LoadTsFileStatement extends Statement {
throw new FileNotFoundException(
String.format(
"Can not find %s on this machine, notice that load can only handle files on this machine.",
- filePath));
+ file.getPath()));
}
- findAllTsFile(file);
+ tsFiles.addAll(findAllTsFile(file));
}
sortTsFiles(tsFiles);
+ return tsFiles;
}
protected LoadTsFileStatement() {
@@ -94,21 +97,24 @@ public class LoadTsFileStatement extends Statement {
this.statementType = StatementType.MULTI_BATCH_INSERT;
}
- private void findAllTsFile(File file) {
+ private static List<File> findAllTsFile(File file) {
final File[] files = file.listFiles();
if (files == null) {
- return;
+ return Collections.emptyList();
}
+
+ final List<File> tsFiles = new ArrayList<>();
for (File nowFile : files) {
if (nowFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) {
tsFiles.add(nowFile);
} else if (nowFile.isDirectory()) {
- findAllTsFile(nowFile);
+ tsFiles.addAll(findAllTsFile(nowFile));
}
}
+ return tsFiles;
}
- private void sortTsFiles(List<File> files) {
+ private static void sortTsFiles(List<File> files) {
files.sort(
(o1, o2) -> {
String file1Name = o1.getName();
@@ -129,6 +135,10 @@ public class LoadTsFileStatement extends Statement {
this.databaseLevel = databaseLevel;
}
+ public void setDatabase(String database) {
+ this.database = database;
+ }
+
public void setVerifySchema(boolean verifySchema) {
this.verifySchema = verifySchema;
}
@@ -153,6 +163,14 @@ public class LoadTsFileStatement extends Statement {
return databaseLevel;
}
+ public String getDatabase() {
+ return database;
+ }
+
+ public String getModel() {
+ return model;
+ }
+
public List<File> getTsFiles() {
return tsFiles;
}
@@ -180,7 +198,11 @@ public class LoadTsFileStatement extends Statement {
private void initAttributes() {
this.databaseLevel = LoadTsFileConfigurator.parseOrGetDefaultDatabaseLevel(loadAttributes);
+ this.database = LoadTsFileConfigurator.parseDatabaseName(loadAttributes);
this.deleteAfterLoad = LoadTsFileConfigurator.parseOrGetDefaultOnSuccess(loadAttributes);
+ this.model =
+ LoadTsFileConfigurator.parseOrGetDefaultModel(
+ loadAttributes, LoadTsFileConfigurator.MODEL_TREE_VALUE);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index 706081ce936..a6b937f31ff 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -2318,9 +2318,6 @@ public class TsFileProcessor {
public void registerToTsFile(
String tableName, Function<String, TableSchema> tableSchemaFunction) {
- getWriter()
- .getKnownSchema()
- .getTableSchemaMap()
- .computeIfAbsent(tableName, tableSchemaFunction);
+ getWriter().getSchema().getTableSchemaMap().computeIfAbsent(tableName, tableSchemaFunction);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java
index d09540133ab..f5f3b8da0c9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java
@@ -24,6 +24,8 @@ import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.commons.lang3.StringUtils;
+import javax.annotation.Nullable;
+
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
@@ -40,6 +42,11 @@ public class LoadTsFileConfigurator {
case ON_SUCCESS_KEY:
validateOnSuccessParam(value);
break;
+ case MODEL_KEY:
+ validateModelParam(value);
+ break;
+ case DATABASE_NAME_KEY:
+ break;
default:
throw new SemanticException("Invalid parameter '" + key + "' for LOAD TSFILE command.");
}
@@ -73,6 +80,12 @@ public class LoadTsFileConfigurator {
DATABASE_LEVEL_KEY, String.valueOf(DATABASE_LEVEL_DEFAULT_VALUE)));
}
+ private static final String DATABASE_NAME_KEY = "database-name";
+
+ public static @Nullable String parseDatabaseName(final Map<String, String> loadAttributes) {
+ return loadAttributes.get(DATABASE_NAME_KEY);
+ }
+
private static final String ON_SUCCESS_KEY = "on-success";
private static final String ON_SUCCESS_DELETE_VALUE = "delete";
private static final String ON_SUCCESS_NONE_VALUE = "none";
@@ -94,6 +107,27 @@ public class LoadTsFileConfigurator {
return StringUtils.isEmpty(value) || ON_SUCCESS_DELETE_VALUE.equals(value);
}
+ public static final String MODEL_KEY = "model";
+ public static final String MODEL_TREE_VALUE = "tree";
+ public static final String MODEL_TABLE_VALUE = "table";
+ public static final Set<String> MODEL_VALUE_SET =
+ Collections.unmodifiableSet(
+ new HashSet<>(Arrays.asList(MODEL_TREE_VALUE, MODEL_TABLE_VALUE)));
+
+ public static void validateModelParam(final String model) {
+ if (!MODEL_VALUE_SET.contains(model)) {
+ throw new SemanticException(
+ String.format(
+ "Given %s value '%s' is not supported, please input a valid value.",
+ MODEL_KEY, model));
+ }
+ }
+
+ public static @Nullable String parseOrGetDefaultModel(
+ final Map<String, String> loadAttributes, final String defaultModel) {
+ return loadAttributes.getOrDefault(MODEL_KEY, defaultModel);
+ }
+
private LoadTsFileConfigurator() {
throw new IllegalStateException("Utility class");
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java
index 849d45c61c8..e425c709815 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java
@@ -39,9 +39,12 @@ public class LoadTsFileNodeTest {
@Test
public void testLoadSingleTsFileNode() {
TsFileResource resource = new TsFileResource(new File("1"));
- LoadSingleTsFileNode node = new LoadSingleTsFileNode(new PlanNodeId(""), resource, true, 0L);
+ String database = "root.db";
+ LoadSingleTsFileNode node =
+ new LoadSingleTsFileNode(new PlanNodeId(""), resource, false, database, true, 0L);
Assert.assertTrue(node.isDeleteAfterLoad());
Assert.assertEquals(resource, node.getTsFileResource());
+ Assert.assertEquals(database, node.getDatabase());
Assert.assertNull(node.getLocalRegionReplicaSet());
Assert.assertNull(node.getRegionReplicaSet());
Assert.assertEquals(Collections.emptyList(), node.getChildren());
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java
index 587c18b0fc0..c1266a614a8 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/DataPartition.java
@@ -231,7 +231,7 @@ public class DataPartition extends Partition {
throw new RuntimeException(
"Database "
+ databaseName
- + "not exists and failed to create automatically because enable_auto_create_schema is FALSE.");
+ + " not exists and failed to create automatically because enable_auto_create_schema is FALSE.");
}
List<TRegionReplicaSet> regions =
databasePartitionMap.get(seriesPartitionSlot).get(timePartitionSlot);
diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
index 1a3c4a5817f..f1d5f14c415 100644
--- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
+++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
@@ -240,9 +240,21 @@ showFunctionsStatement
// -------------------------------------------- Load Statement ---------------------------------------------------------
loadTsFileStatement
- : LOAD fileName=string properties?
+ : LOAD fileName=string (loadFileWithAttributeClauses)?
;
+loadFileWithAttributeClauses
+ : WITH
+ '('
+ (loadFileWithAttributeClause ',')* loadFileWithAttributeClause?
+ ')'
+ ;
+
+loadFileWithAttributeClause
+ : loadFileWithKey=STRING EQ loadFileWithValue=STRING
+ ;
+
+
// -------------------------------------------- Pipe Statement ---------------------------------------------------------
createPipeStatement