You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2021/07/27 11:31:08 UTC
[iotdb] branch master updated: [IOTDB-1519]add metadata checking
option and change grammar for loading tsfile (#3576)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 716710c [IOTDB-1519]add metadata checking option and change grammar for loading tsfile (#3576)
716710c is described below
commit 716710c4466e4c9f4eafc8edb38cf8b6ac489c40
Author: yschengzi <87...@users.noreply.github.com>
AuthorDate: Tue Jul 27 19:30:46 2021 +0800
[IOTDB-1519]add metadata checking option and change grammar for loading tsfile (#3576)
---
.../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 21 +++-
.../cluster/log/snapshot/FileSnapshotTest.java | 2 +-
.../UserGuide/System-Tools/Load-External-Tsfile.md | 32 +++--
.../UserGuide/System-Tools/Load-External-Tsfile.md | 56 +++++----
.../level/LevelCompactionTsFileManagement.java | 2 +-
.../engine/storagegroup/StorageGroupProcessor.java | 131 ++++++++++-----------
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 36 ++++++
.../iotdb/db/qp/logical/sys/LoadFilesOperator.java | 25 +++-
.../iotdb/db/qp/physical/sys/OperateFilePlan.java | 23 +++-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 50 +++++---
.../db/integration/IoTDBLoadExternalTsfileIT.java | 121 ++++++++++++++++++-
.../iotdb/db/qp/physical/PhysicalPlanTest.java | 42 +++++--
.../iotdb/db/utils/TsFileRewriteToolTest.java | 4 +-
13 files changed, 404 insertions(+), 141 deletions(-)
diff --git a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4
index f3d1c5a..0943224 100644
--- a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4
+++ b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4
@@ -91,7 +91,7 @@ statement
| COUNT STORAGE GROUP prefixPath? #countStorageGroup
| COUNT NODES prefixPath LEVEL OPERATOR_EQ INT #countNodes
| LOAD CONFIGURATION (MINUS GLOBAL)? #loadConfigurationStatement
- | LOAD stringLiteral autoCreateSchema?#loadFiles
+ | LOAD stringLiteral loadFilesClause?#loadFiles
| REMOVE stringLiteral #removeFile
| MOVE stringLiteral stringLiteral #moveFile
| DELETE PARTITION prefixPath INT(COMMA INT)* #deletePartition
@@ -687,9 +687,10 @@ property
: name=ID OPERATOR_EQ value=propertyValue
;
-autoCreateSchema
- : booleanClause
- | booleanClause INT
+loadFilesClause
+ : AUTOREGISTER OPERATOR_EQ booleanClause (COMMA loadFilesClause)?
+ | SGLEVEL OPERATOR_EQ INT (COMMA loadFilesClause)?
+ | VERIFY OPERATOR_EQ booleanClause (COMMA loadFilesClause)?
;
triggerEventClause
@@ -1184,6 +1185,18 @@ SCHEMA
: S C H E M A
;
+AUTOREGISTER
+ : A U T O R E G I S T E R
+ ;
+
+VERIFY
+ : V E R I F Y
+ ;
+
+SGLEVEL
+ : S G L E V E L
+ ;
+
TEMPORARY
: T E M P O R A R Y
;
diff --git a/cluster/src/test/java/org/apache/iotdb/cluster/log/snapshot/FileSnapshotTest.java b/cluster/src/test/java/org/apache/iotdb/cluster/log/snapshot/FileSnapshotTest.java
index 2a88719..e08798f 100644
--- a/cluster/src/test/java/org/apache/iotdb/cluster/log/snapshot/FileSnapshotTest.java
+++ b/cluster/src/test/java/org/apache/iotdb/cluster/log/snapshot/FileSnapshotTest.java
@@ -295,7 +295,7 @@ public class FileSnapshotTest extends DataSnapshotTest {
}
StorageGroupProcessor processor =
StorageEngine.getInstance().getProcessor(new PartialPath(TestUtils.getTestSg(0)));
- assertEquals(9, processor.getPartitionMaxFileVersions(0));
+ assertEquals(10, processor.getPartitionMaxFileVersions(0));
List<TsFileResource> loadedFiles = processor.getSequenceFileTreeSet();
assertEquals(tsFileResources.size(), loadedFiles.size());
for (int i = 0; i < 9; i++) {
diff --git a/docs/UserGuide/System-Tools/Load-External-Tsfile.md b/docs/UserGuide/System-Tools/Load-External-Tsfile.md
index cb1367d..65ada84 100644
--- a/docs/UserGuide/System-Tools/Load-External-Tsfile.md
+++ b/docs/UserGuide/System-Tools/Load-External-Tsfile.md
@@ -31,33 +31,45 @@ The user sends specified commands to the Apache IoTDB system through the Cli too
#### load tsfiles
-The command to load tsfiles is `load "<path/dir>" [true/false] [storage group level]`.
+The command to load tsfiles is `load "<path/dir>" [autoregister=true/false][,sglevel=int][,verify=true/false]`.
This command has two usages:
1. Load a single tsfile by specifying a file path (absolute path).
-The second parameter indicates the path of the tsfile to be loaded and the name of the tsfile needs to conform to the tsfile naming convention, that is, `{systemTime}-{versionNum}-{mergeNum} .tsfile`. The third and fourth parameters are optional. If the metadata correspond to the timeseries in the tsfile to be loaded does not exist, you can choose whether to create the schema automatically. If the third parameter is true, the schema is created automatically. If the third parameter is fal [...]
+The second parameter indicates the path of the tsfile to be loaded and the name of the tsfile needs to conform to the tsfile naming convention, that is, `{systemTime}-{versionNum}-{in_space_compaction_num}-{cross_space_compaction_num}.tsfile`. This command has three options: autoregister, sglevel and verify.
+
+AUTOREGISTER option. If the metadata correspond to the timeseries in the tsfile to be loaded does not exist, you can choose whether to create the schema automatically. If this parameter is true, the schema is created automatically. If it is false, the schema will not be created. By default, the schema will be created.
+
+SGLEVEL option. If the storage group correspond to the tsfile does not exist, the user can set the level of storage group through the fourth parameter. By default, it uses the storage group level which is set in `iotdb-engine.properties`.
+
+VERIFY option. If this parameter is true, All timeseries in this loading tsfile will be compared with the timeseries in IoTDB. If existing a measurement which has different datatype with the measurement in IoTDB, the loading process will be stopped and exit. If consistence can be promised, setting false for this parameter will be a better choice.
+
If the `.resource` file corresponding to the file exists, it will be loaded into the data directory and engine of the Apache IoTDB. Otherwise, the corresponding `.resource` file will be regenerated from the tsfile file.
Examples:
* `load "/Users/Desktop/data/1575028885956-101-0.tsfile"`
-* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" false`
-* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" true`
-* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" true 1`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" autoregister=false`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" autoregister=true`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" sglevel=1`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" verify=true`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" autoregister=true,sglevel=1`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" verify=false,sglevel=1`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" autoregister=false,verify=true`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" autoregister=false,sglevel=1,verify=true`
2. Load a batch of files by specifying a folder path (absolute path).
-The second parameter indicates the path of the tsfile to be loaded and the name of the tsfiles need to conform to the tsfile naming convention, that is, `{systemTime}-{versionNum}-{mergeNum} .tsfile`. The third and fourth parameters are optional. When the metadata corresponding to the time series in the tsfile to be loaded does not exist, you can choose whether to create the schema automatically. If the third parameter is true, the schema will be created automatically. If the third param [...]
-If the `.resource` file corresponding to the file exists, they will be loaded into the data directory and engine of the Apache IoTDB. Otherwise, the corresponding` .resource` files will be regenerated from the tsfile sfile.
+The second parameter indicates the path of the tsfile to be loaded and the name of the tsfiles need to conform to the tsfile naming convention, that is, `{systemTime}-{versionNum}-{in_space_compaction_num}-{cross_space_compaction_num}.tsfile`. The options above also works for this command.
Examples:
* `load "/Users/Desktop/data"`
-* `load "/Users/Desktop/data" false`
-* `load "/Users/Desktop/data" true`
-* `load "/Users/Desktop/data" true 1`
+* `load "/Users/Desktop/data" autoregister=false`
+* `load "/Users/Desktop/data" autoregister=true`
+* `load "/Users/Desktop/data" autoregister=true,sglevel=1`
+* `load "/Users/Desktop/data" autoregister=false,sglevel=1,verify=true`
#### remove a tsfile
diff --git a/docs/zh/UserGuide/System-Tools/Load-External-Tsfile.md b/docs/zh/UserGuide/System-Tools/Load-External-Tsfile.md
index f8e33d1..20e3772 100644
--- a/docs/zh/UserGuide/System-Tools/Load-External-Tsfile.md
+++ b/docs/zh/UserGuide/System-Tools/Load-External-Tsfile.md
@@ -24,54 +24,68 @@
## 加载 TsFile
### 介绍
-加载外部tsfile文件工具允许用户向正在运行中的Apache IoTDB中加载、删除或移出tsfile文件。
+加载外部 tsfile 文件工具允许用户向正在运行中的 Apache IoTDB 中加载、删除或移出 tsfile 文件。
### 使用方式
-用户通过Cli工具或JDBC向Apache IoTDB系统发送指定命令实现文件加载的功能。
+用户通过 Cli 工具或 JDBC 向 Apache IoTDB 系统发送指定命令实现文件加载的功能。
-#### 加载tsfile文件
-加载tsfile文件的指令为：`load "<path/dir>" [true/false] [storage group level]`
+#### 加载 tsfile 文件
+加载 tsfile 文件的指令为：`load "<path/dir>" [autoregister=true/false][,sglevel=int][,verify=true/false]`
该指令有两种用法：
-1. 通过指定文件路径(绝对路径)加载单tsfile文件。
+1. 通过指定文件路径(绝对路径)加载单 tsfile 文件。
-第二个参数表示待加载的tsfile文件的路径，其中文件名称需要符合tsfile的命名规范，即`{systemTime}-{versionNum}-{mergeNum}.tsfile`。第三、四个参数为可选项。当待加载的tsfile文件中时间序列对应的元数据不存在时，用户可以选择是否自动创建schema，参数为true表示自动创建schema，相反false表示不创建，缺省时默认创建schema。当tsfile对应的存储组不存在时，用户可以通过第四个参数来制定存储组的级别，默认为`iotdb-engine.properties`中设置的级别。若待加载的tsfile文件对应的`.resource`文件存在，会被一并加载至Apache IoTDB数据文件的目录和引擎中，否则将通过tsfile文件重新生成对应的`.resource`文件，即加载的tsfile文件所对应的`.resource`文件不是必要的。
+第二个参数表示待加载的 tsfile 文件的路径，其中文件名称需要符合 tsfile 的命名规范，即`{systemTime}-{versionNum}-{in_space_compaction_num}-{cross_space_compaction_num}.tsfile`。load 命令有三个可选项，分别是 autoregister，值域为 true/false，sglevel，值域为整数，verify，值域为 true/false。不同选项之间用逗号连接，选项之间无顺序要求。
+
+AUTOREGISTER 选项表示当待加载的 tsfile 文件中时间序列对应的元数据不存在时，用户可以选择是否自动创建 schema ，参数为 true 表示自动创建 schema，相反 false 表示不创建，缺省时默认创建 schema。
+
+SGLEVEL 选项，当 tsfile 对应的存储组不存在时，用户可以通过 sglevel 参数的值来制定存储组的级别，默认为`iotdb-engine.properties`中设置的级别。例如当设置 level 参数为1时表明此 tsfile 中所有时间序列中层级为1的前缀路径是存储组，即若存在设备 root.sg.d1.s1，此时 root.sg 被指定为存储组。
+
+VERIFY 选项表示是否对载入的 tsfile 中的所有时间序列进行元数据检查，默认为 true。开启时，若载入的 tsfile 中的时间序列在当前 iotdb 中也存在，则会比较该时间序列的所有 Measurement 的数据类型是否一致，如果出现不一致将会导致载入失败，关闭该选项会跳过检查，载入更快。
+
+若待加载的 tsfile 文件对应的`.resource`文件存在，会被一并加载至 Apache IoTDB 数据文件的目录和引擎中，否则将通过 tsfile 文件重新生成对应的`.resource`文件，即加载的 tsfile 文件所对应的`.resource`文件不是必要的。
示例：
-* load `"/Users/Desktop/data/1575028885956-101-0.tsfile"`
-* load `"/Users/Desktop/data/1575028885956-101-0.tsfile" false`
-* load `"/Users/Desktop/data/1575028885956-101-0.tsfile" true`
-* load `"/Users/Desktop/data/1575028885956-101-0.tsfile" true 1`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile"`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" autoregister=false`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" autoregister=true`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" sglevel=1`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" verify=true`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" autoregister=true,sglevel=1`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" verify=false,sglevel=1`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" autoregister=false,verify=true`
+* `load "/Users/Desktop/data/1575028885956-101-0.tsfile" autoregister=false,sglevel=1,verify=true`
2. 通过指定文件夹路径(绝对路径)批量加载文件。
-第二个参数表示待加载的tsfile文件的路径，其中文件名称需要符合tsfile的命名规范，即`{systemTime}-{versionNum}-{mergeNum}.tsfile`。第三、四个参数为可选项。当待加载的tsfile文件中时间序列对应的元数据不存在时，用户可以选择是否自动创建schema，参数为true表示自动创建schema，相反false表示不创建，缺省时默认创建schema。当tsfile对应的存储组不存在时，用户可以通过第四个参数来制定存储组的级别，默认为`iotdb-engine.properties`中设置的级别。若待加载文件对应的`.resource`文件存在，则会一并加载至Apache IoTDB数据文件目录和引擎中，否则将通过tsfile文件重新生成对应的`.resource`文件，即加载的tsfile文件所对应的`.resource`文件不是必要的。
+第二个参数表示待加载的 tsfile 文件夹的路径，其中文件夹内所有文件名称需要符合 tsfile 的命名规范，即`{systemTime}-{versionNum}-{in_space_compaction_num}-{cross_space_compaction_num}.tsfile`。选项意义与加载单个 tsfile 文件相同。
示例：
-* load `"/Users/Desktop/data"`
-* load `"/Users/Desktop/data" false`
-* load `"/Users/Desktop/data" true`
-* load `"/Users/Desktop/data" true 1`
+* `load "/Users/Desktop/data"`
+* `load "/Users/Desktop/data" autoregister=false`
+* `load "/Users/Desktop/data" autoregister=true`
+* `load "/Users/Desktop/data" autoregister=true,sglevel=1`
+* `load "/Users/Desktop/data" autoregister=false,sglevel=1,verify=true`
-#### 删除tsfile文件
+#### 删除 tsfile 文件
-删除tsfile文件的指令为：`remove "<path>"`
+删除 tsfile 文件的指令为：`remove "<path>"`
-该指令通过指定文件路径删除tsfile文件，具体做法是将该tsfile和其对应的`.resource`和`.modification`文件全部删除。
+该指令通过指定文件路径删除 tsfile 文件，具体做法是将该 tsfile 和其对应的`.resource`和`.modification`文件全部删除。
示例：
* `remove "root.vehicle/1575028885956-101-0.tsfile"`
* `remove "1575028885956-101-0.tsfile"`
-#### 移出tsfile文件至指定目录
+#### 移出 tsfile 文件至指定目录
-移出tsfile文件的指令为：`move "<path>" "<dir>"`
+移出 tsfile 文件的指令为：`move "<path>" "<dir>"`
-该指令将指定路径的tsfile文件移动至目标文件夹(绝对路径)中，具体做法是在引擎中移出该tsfile，并将该tsfile文件和其对应的`.resource`文件移动到目标文件夹下
+该指令将指定路径的 tsfile 文件移动至目标文件夹(绝对路径)中，具体做法是在引擎中移出该 tsfile，并将该 tsfile 文件和其对应的`.resource`文件移动到目标文件夹下
示例：
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
index afefcd5..c7b0354 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java
@@ -195,7 +195,7 @@ public class LevelCompactionTsFileManagement extends TsFileManagement {
}
} else {
List<List<TsFileResource>> unSequenceTsFileList =
- unSequenceTsFileResources.get(timePartition);
+ unSequenceTsFileResources.getOrDefault(timePartition, new ArrayList<>());
for (int i = unSequenceTsFileList.size() - 1; i >= 0; i--) {
result.addAll(unSequenceTsFileList.get(i));
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index d124341..e69982b 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -1304,11 +1304,11 @@ public class StorageGroupProcessor {
private String getNewTsFileName(long timePartitionId) {
long version = partitionMaxFileVersions.getOrDefault(timePartitionId, 0L) + 1;
partitionMaxFileVersions.put(timePartitionId, version);
- return getNewTsFileName(System.currentTimeMillis(), version, 0);
+ return getNewTsFileName(System.currentTimeMillis(), version, 0, 0);
}
- private String getNewTsFileName(long time, long version, int mergeCnt) {
- return TsFileResource.getNewTsFileName(System.currentTimeMillis(), version, 0, 0);
+ private String getNewTsFileName(long time, long version, int mergeCnt, int unSeqMergeCnt) {
+ return TsFileResource.getNewTsFileName(time, version, mergeCnt, unSeqMergeCnt);
}
/**
@@ -2315,42 +2315,37 @@ public class StorageGroupProcessor {
List<TsFileResource> sequenceList = tsFileManagement.getTsFileList(true);
int insertPos = findInsertionPosition(newTsFileResource, newFilePartitionId, sequenceList);
- if (insertPos == POS_ALREADY_EXIST) {
- return;
- }
+ String newFileName, renameInfo;
+ LoadTsFileType tsFileType;
// loading tsfile by type
if (insertPos == POS_OVERLAP) {
- loadTsFileByType(
- LoadTsFileType.LOAD_UNSEQUENCE,
- tsfileToBeInserted,
- newTsFileResource,
- newFilePartitionId);
+ newFileName =
+ getNewTsFileName(
+ System.currentTimeMillis(),
+ getAndSetNewVersion(newFilePartitionId, newTsFileResource),
+ 0,
+ 0);
+ renameInfo = IoTDBConstant.UNSEQUENCE_FLODER_NAME;
+ tsFileType = LoadTsFileType.LOAD_UNSEQUENCE;
+ newTsFileResource.setSeq(false);
} else {
-
// check whether the file name needs to be renamed.
- if (!tsFileManagement.isEmpty(true)) {
- String newFileName =
- getFileNameForLoadingFile(
- tsfileToBeInserted.getName(),
- insertPos,
- newTsFileResource.getTimePartition(),
- sequenceList);
- if (!newFileName.equals(tsfileToBeInserted.getName())) {
- logger.info(
- "Tsfile {} must be renamed to {} for loading into the sequence list.",
- tsfileToBeInserted.getName(),
- newFileName);
- newTsFileResource.setFile(
- fsFactory.getFile(tsfileToBeInserted.getParentFile(), newFileName));
- }
- }
- loadTsFileByType(
- LoadTsFileType.LOAD_SEQUENCE,
- tsfileToBeInserted,
- newTsFileResource,
- newFilePartitionId);
+ newFileName = getFileNameForSequenceLoadingFile(insertPos, newTsFileResource, sequenceList);
+ renameInfo = IoTDBConstant.SEQUENCE_FLODER_NAME;
+ tsFileType = LoadTsFileType.LOAD_SEQUENCE;
+ newTsFileResource.setSeq(true);
+ }
+
+ if (!newFileName.equals(tsfileToBeInserted.getName())) {
+ logger.info(
+ "TsFile {} must be renamed to {} for loading into the " + renameInfo + " list.",
+ tsfileToBeInserted.getName(),
+ newFileName);
+ newTsFileResource.setFile(
+ fsFactory.getFile(tsfileToBeInserted.getParentFile(), newFileName));
}
+ loadTsFileByType(tsFileType, tsfileToBeInserted, newTsFileResource, newFilePartitionId);
resetLastCacheWhenLoadingTsfile(newTsFileResource);
// update latest time map
@@ -2399,7 +2394,6 @@ public class StorageGroupProcessor {
TsFileResource newTsFileResource,
long newFilePartitionId,
List<TsFileResource> sequenceList) {
- File tsfileToBeInserted = newTsFileResource.getTsFile();
int insertPos = -1;
@@ -2407,15 +2401,9 @@ public class StorageGroupProcessor {
for (int i = 0; i < sequenceList.size(); i++) {
TsFileResource localFile = sequenceList.get(i);
long localPartitionId = Long.parseLong(localFile.getTsFile().getParentFile().getName());
- if (localPartitionId == newFilePartitionId
- && localFile.getTsFile().getName().equals(tsfileToBeInserted.getName())) {
- return POS_ALREADY_EXIST;
- }
- if (i == sequenceList.size() - 1 && localFile.endTimeEmpty()
- || newFilePartitionId > localPartitionId) {
- // skip files that are in the previous partition and the last empty file, as the all data
- // in those files must be older than the new file
+ if (newFilePartitionId > localPartitionId) {
+ insertPos = i;
continue;
}
@@ -2560,36 +2548,31 @@ public class StorageGroupProcessor {
/**
* Get an appropriate filename to ensure the order between files. The tsfile is named after
- * ({systemTime}-{versionNum}-{mergeNum}.tsfile).
+ * ({systemTime}-{versionNum}-{in_space_compaction_num}-{cross_space_compaction_num}.tsfile).
*
* <p>The sorting rules for tsfile names @see {@link this#compareFileName}, we can restore the
* list based on the file name and ensure the correctness of the order, so there are three cases.
*
- * <p>1. The tsfile is to be inserted in the first place of the list. If the timestamp in the file
- * name is less than the timestamp in the file name of the first tsfile in the list, then the file
- * name is legal and the file name is returned directly. Otherwise, its timestamp can be set to
- * half of the timestamp value in the file name of the first tsfile in the list , and the version
- * number is the version number in the file name of the first tsfile in the list.
+ * <p>1. The tsfile is to be inserted in the first place of the list. Timestamp can be set to half
+ * of the timestamp value in the file name of the first tsfile in the list , and the version
+ * number will be updated to the largest number in this time partition.
*
- * <p>2. The tsfile is to be inserted in the last place of the list. If the timestamp in the file
- * name is lager than the timestamp in the file name of the last tsfile in the list, then the file
- * name is legal and the file name is returned directly. Otherwise, the file name is generated by
+ * <p>2. The tsfile is to be inserted in the last place of the list. The file name is generated by
* the system according to the naming rules and returned.
*
- * <p>3. This file is inserted between two files. If the timestamp in the name of the file
- * satisfies the timestamp between the timestamps in the name of the two files, then it is a legal
- * name and returns directly; otherwise, the time stamp is the mean of the timestamps of the two
- * files, the version number is the version number in the tsfile with a larger timestamp.
+ * <p>3. This file is inserted between two files. The time stamp is the mean of the timestamps of
+ * the two files, the version number will be updated to the largest number in this time partition.
*
- * @param tsfileName origin tsfile name
* @param insertIndex the new file will be inserted between the files [insertIndex, insertIndex +
* 1]
* @return appropriate filename
*/
- private String getFileNameForLoadingFile(
- String tsfileName, int insertIndex, long timePartitionId, List<TsFileResource> sequenceList) {
- long currentTsFileTime = Long.parseLong(tsfileName.split(FILE_NAME_SEPARATOR)[0]);
- long preTime;
+ private String getFileNameForSequenceLoadingFile(
+ int insertIndex, TsFileResource newTsFileResource, List<TsFileResource> sequenceList)
+ throws LoadFileException {
+ long timePartitionId = newTsFileResource.getTimePartition();
+ long preTime, subsequenceTime;
+
if (insertIndex == -1) {
preTime = 0L;
} else {
@@ -2597,22 +2580,26 @@ public class StorageGroupProcessor {
preTime = Long.parseLong(preName.split(FILE_NAME_SEPARATOR)[0]);
}
if (insertIndex == tsFileManagement.size(true) - 1) {
- if (preTime < currentTsFileTime) {
- return tsfileName;
- } else {
- return getNewTsFileName(timePartitionId);
- }
+ subsequenceTime = preTime + ((System.currentTimeMillis() - preTime) << 1);
+ } else {
+ String subsequenceName = sequenceList.get(insertIndex + 1).getTsFile().getName();
+ subsequenceTime = Long.parseLong(subsequenceName.split(FILE_NAME_SEPARATOR)[0]);
}
- String subsequenceName = sequenceList.get(insertIndex + 1).getTsFile().getName();
- long subsequenceTime = Long.parseLong(subsequenceName.split(FILE_NAME_SEPARATOR)[0]);
- long subsequenceVersion = Long.parseLong(subsequenceName.split(FILE_NAME_SEPARATOR)[1]);
- if (preTime < currentTsFileTime && currentTsFileTime < subsequenceTime) {
- return tsfileName;
+ long meanTime = preTime + ((subsequenceTime - preTime) >> 1);
+ if (insertIndex != tsFileManagement.size(true) - 1 && meanTime == subsequenceTime) {
+ throw new LoadFileException("can not load TsFile because of can not find suitable location");
}
- return TsFileResource.getNewTsFileName(
- preTime + ((subsequenceTime - preTime) >> 1), subsequenceVersion, 0, 0);
+ return getNewTsFileName(
+ meanTime, getAndSetNewVersion(timePartitionId, newTsFileResource), 0, 0);
+ }
+
+ private long getAndSetNewVersion(long timePartitionId, TsFileResource tsFileResource) {
+ long version = partitionMaxFileVersions.getOrDefault(timePartitionId, -1L) + 1;
+ partitionMaxFileVersions.put(timePartitionId, version);
+ tsFileResource.setVersion(version);
+ return version;
}
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index ebb26e6..2690392 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -140,6 +140,7 @@ import org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Field;
@@ -1097,6 +1098,12 @@ public class PlanExecutor implements IPlanExecutor {
List<ChunkGroupMetadata> chunkGroupMetadataList = new ArrayList<>();
try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), false)) {
reader.selfCheck(schemaMap, chunkGroupMetadataList, false);
+ if (plan.getVerifyMetadata()) {
+ loadNewTsFileVerifyMetadata(reader);
+ }
+ } catch (IOException e) {
+ logger.warn("can not get timeseries metadata from {}.", file.getAbsoluteFile());
+ throw new QueryProcessException(e.getMessage());
}
FileLoaderUtils.checkTsFileResource(tsFileResource);
@@ -1135,6 +1142,35 @@ public class PlanExecutor implements IPlanExecutor {
}
}
+ private void loadNewTsFileVerifyMetadata(TsFileSequenceReader tsFileSequenceReader)
+ throws MetadataException, QueryProcessException, IOException {
+ Map<String, List<TimeseriesMetadata>> metadataSet =
+ tsFileSequenceReader.getAllTimeseriesMetadata();
+ for (Map.Entry<String, List<TimeseriesMetadata>> entry : metadataSet.entrySet()) {
+ String deviceId = entry.getKey();
+ PartialPath devicePath = new PartialPath(deviceId);
+ if (!IoTDB.metaManager.isPathExist(devicePath)) {
+ continue;
+ }
+ for (TimeseriesMetadata metadata : entry.getValue()) {
+ PartialPath fullPath =
+ new PartialPath(deviceId + TsFileConstant.PATH_SEPARATOR + metadata.getMeasurementId());
+ if (IoTDB.metaManager.isPathExist(fullPath)) {
+ TSDataType dataType = IoTDB.metaManager.getSeriesSchema(fullPath).getType();
+ if (dataType != metadata.getTSDataType()) {
+ throw new QueryProcessException(
+ fullPath.getFullPath()
+ + " is "
+ + metadata.getTSDataType().name()
+ + " in the loading TsFile but is "
+ + dataType.name()
+ + " in IoTDB.");
+ }
+ }
+ }
+ }
+ }
+
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
private void createSchemaAutomatically(
List<ChunkGroupMetadata> chunkGroupMetadataList,
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/LoadFilesOperator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/LoadFilesOperator.java
index d74e118..5a860b6 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/LoadFilesOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/LoadFilesOperator.java
@@ -27,17 +27,25 @@ import org.apache.iotdb.db.qp.strategy.PhysicalGenerator;
import java.io.File;
+/**
+ * operator for loading tsfile including four property: file, the loading file. autoCreateSchema,
+ * need auto create schema or not. sglevel, the level of sg. metadataCheck, need check for metadata
+ * or not.
+ */
public class LoadFilesOperator extends Operator {
private File file;
private boolean autoCreateSchema;
private int sgLevel;
+ private boolean verifyMetadata;
- public LoadFilesOperator(File file, boolean autoCreateSchema, int sgLevel) {
+ public LoadFilesOperator(
+ File file, boolean autoCreateSchema, int sgLevel, boolean verifyMetadata) {
super(SQLConstant.TOK_LOAD_FILES);
this.file = file;
this.autoCreateSchema = autoCreateSchema;
this.sgLevel = sgLevel;
+ this.verifyMetadata = verifyMetadata;
this.operatorType = OperatorType.LOAD_FILES;
}
@@ -53,9 +61,22 @@ public class LoadFilesOperator extends Operator {
return sgLevel;
}
+ public void setAutoCreateSchema(boolean autoCreateSchema) {
+ this.autoCreateSchema = autoCreateSchema;
+ }
+
+ public void setSgLevel(int sgLevel) {
+ this.sgLevel = sgLevel;
+ }
+
+ public void setVerifyMetadata(boolean verifyMetadata) {
+ this.verifyMetadata = verifyMetadata;
+ }
+
@Override
public PhysicalPlan generatePhysicalPlan(PhysicalGenerator generator)
throws QueryProcessException {
- return new OperateFilePlan(file, OperatorType.LOAD_FILES, autoCreateSchema, sgLevel);
+ return new OperateFilePlan(
+ file, OperatorType.LOAD_FILES, autoCreateSchema, sgLevel, verifyMetadata);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/OperateFilePlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/OperateFilePlan.java
index bb47daf..de23250 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/OperateFilePlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/OperateFilePlan.java
@@ -32,18 +32,33 @@ public class OperateFilePlan extends PhysicalPlan {
private File targetDir;
private boolean autoCreateSchema;
private int sgLevel;
+ private boolean verifyMetadata;
public OperateFilePlan(File file, OperatorType operatorType) {
super(false, operatorType);
this.file = file;
}
+ /**
+ * used for generate loading tsfile physical plan.
+ *
+ * @param file the loading file
+ * @param operatorType the operator type
+ * @param autoCreateSchema auto create schema if needed
+ * @param sgLevel the level of sg
+ * @param verifyMetadata metadata check if needed
+ */
public OperateFilePlan(
- File file, OperatorType operatorType, boolean autoCreateSchema, int sgLevel) {
+ File file,
+ OperatorType operatorType,
+ boolean autoCreateSchema,
+ int sgLevel,
+ boolean verifyMetadata) {
super(false, operatorType);
this.file = file;
this.autoCreateSchema = autoCreateSchema;
this.sgLevel = sgLevel;
+ this.verifyMetadata = verifyMetadata;
}
public OperateFilePlan(File file, File targetDir, OperatorType operatorType) {
@@ -73,6 +88,10 @@ public class OperateFilePlan extends PhysicalPlan {
return sgLevel;
}
+ public boolean getVerifyMetadata() {
+ return verifyMetadata;
+ }
+
@Override
public String toString() {
return "OperateFilePlan{"
@@ -84,6 +103,8 @@ public class OperateFilePlan extends PhysicalPlan {
+ autoCreateSchema
+ ", sgLevel="
+ sgLevel
+ + ", verify="
+ + verifyMetadata
+ ", operatorType="
+ getOperatorType()
+ '}';
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java b/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
index bcd6a06..6da499b 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
@@ -171,6 +171,7 @@ import org.apache.iotdb.db.qp.sql.SqlBaseParser.ListRolePrivilegesContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.ListUserContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.ListUserPrivilegesContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.LoadConfigurationStatementContext;
+import org.apache.iotdb.db.qp.sql.SqlBaseParser.LoadFilesClauseContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.LoadFilesContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.LoadStatementContext;
import org.apache.iotdb.db.qp.sql.SqlBaseParser.MeasurementNameContext;
@@ -981,24 +982,16 @@ public class IoTDBSqlVisitor extends SqlBaseBaseVisitor<Operator> {
@Override
public Operator visitLoadFiles(LoadFilesContext ctx) {
- if (ctx.autoCreateSchema() != null) {
- if (ctx.autoCreateSchema().INT() != null) {
- return new LoadFilesOperator(
+ LoadFilesOperator loadFilesOperator =
+ new LoadFilesOperator(
new File(removeStringQuote(ctx.stringLiteral().getText())),
- Boolean.parseBoolean(ctx.autoCreateSchema().booleanClause().getText()),
- Integer.parseInt(ctx.autoCreateSchema().INT().getText()));
- } else {
- return new LoadFilesOperator(
- new File(removeStringQuote(ctx.stringLiteral().getText())),
- Boolean.parseBoolean(ctx.autoCreateSchema().booleanClause().getText()),
- IoTDBDescriptor.getInstance().getConfig().getDefaultStorageGroupLevel());
- }
- } else {
- return new LoadFilesOperator(
- new File(removeStringQuote(ctx.stringLiteral().getText())),
- true,
- IoTDBDescriptor.getInstance().getConfig().getDefaultStorageGroupLevel());
+ true,
+ IoTDBDescriptor.getInstance().getConfig().getDefaultStorageGroupLevel(),
+ true);
+ if (ctx.loadFilesClause() != null) {
+ parseLoadFiles(loadFilesOperator, ctx.loadFilesClause());
}
+ return loadFilesOperator;
}
@Override
@@ -2406,4 +2399,29 @@ public class IoTDBSqlVisitor extends SqlBaseBaseVisitor<Operator> {
timestampStr));
}
}
+
+ /**
+ * used for parsing load tsfile, context will be one of "SCHEMA, LEVEL, METADATA", and maybe
+ * followed by a recursion property statement
+ *
+ * @param operator the result operator, setting by clause context
+ * @param ctx context of property statement
+ */
+ private void parseLoadFiles(LoadFilesOperator operator, LoadFilesClauseContext ctx) {
+ if (ctx.AUTOREGISTER() != null) {
+ operator.setAutoCreateSchema(Boolean.parseBoolean(ctx.booleanClause().getText()));
+ } else if (ctx.SGLEVEL() != null) {
+ operator.setSgLevel(Integer.parseInt(ctx.INT().getText()));
+ } else if (ctx.VERIFY() != null) {
+ operator.setVerifyMetadata(Boolean.parseBoolean(ctx.booleanClause().getText()));
+ } else {
+ throw new SQLParserException(
+ String.format(
+ "load tsfile format %s error, please input AUTOREGISTER | SGLEVEL | VERIFY.",
+ ctx.getText()));
+ }
+ if (ctx.loadFilesClause() != null) {
+ parseLoadFiles(operator, ctx.loadFilesClause());
+ }
+ }
}
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
index f1a149f..9a6c9a8 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
@@ -52,6 +52,7 @@ import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class IoTDBLoadExternalTsfileIT {
@@ -616,7 +617,8 @@ public class IoTDBLoadExternalTsfileIT {
// test not load metadata automatically, it will occur errors.
boolean hasError = false;
try {
- statement.execute(String.format("load \"%s\" false 1", tmpDir.getAbsolutePath()));
+ statement.execute(
+ String.format("load \"%s\" autoregister=false,sglevel=1", tmpDir.getAbsolutePath()));
} catch (Exception e) {
hasError = true;
}
@@ -624,7 +626,8 @@ public class IoTDBLoadExternalTsfileIT {
// test load metadata automatically, it will succeed.
tmpDir = tmpDir.getParentFile().getParentFile().getParentFile();
- statement.execute(String.format("load \"%s\" true 1", tmpDir.getAbsolutePath()));
+ statement.execute(
+ String.format("load \"%s\" autoregister=true,sglevel=1", tmpDir.getAbsolutePath()));
resources =
new ArrayList<>(
StorageEngine.getInstance()
@@ -648,6 +651,120 @@ public class IoTDBLoadExternalTsfileIT {
}
@Test
+ public void loadTsFileTestWithVerifyMetadata() throws Exception {
+ try (Connection connection =
+ DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ List<TsFileResource> resources =
+ new ArrayList<>(
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.vehicle"))
+ .getSequenceFileTreeSet());
+ assertEquals(1, resources.size());
+ File vehicleTmpDir =
+ new File(
+ resources
+ .get(0)
+ .getTsFile()
+ .getParentFile()
+ .getParentFile()
+ .getParentFile()
+ .getParentFile(),
+ "tmp" + File.separator + "root.vehicle");
+ if (!vehicleTmpDir.exists()) {
+ vehicleTmpDir.mkdirs();
+ }
+
+ for (TsFileResource resource : resources) {
+ statement.execute("move \"" + resource.getTsFilePath() + "\" \"" + vehicleTmpDir + "\"");
+ }
+
+ resources =
+ new ArrayList<>(
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.test"))
+ .getSequenceFileTreeSet());
+ assertEquals(2, resources.size());
+
+ File testTmpDir = new File(vehicleTmpDir.getParentFile(), "root.test");
+ if (!testTmpDir.exists()) {
+ testTmpDir.mkdirs();
+ }
+
+ for (TsFileResource resource : resources) {
+ statement.execute("move \"" + resource.getTsFilePath() + "\" \"" + testTmpDir + "\"");
+ }
+
+ for (String sql : deleteSqls) {
+ statement.execute(sql);
+ }
+
+ List<String> metaDataSqls =
+ new ArrayList<>(
+ Arrays.asList(
+ "SET STORAGE GROUP TO root.vehicle",
+ "SET STORAGE GROUP TO root.test",
+ "CREATE TIMESERIES root.vehicle.d0.s0 WITH DATATYPE=INT64, ENCODING=RLE",
+ "CREATE TIMESERIES root.vehicle.d0.s1 WITH DATATYPE=TEXT, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.vehicle.d1.s2 WITH DATATYPE=FLOAT, ENCODING=RLE",
+ "CREATE TIMESERIES root.vehicle.d1.s3 WITH DATATYPE=BOOLEAN, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.test.d0.s0 WITH DATATYPE=FLOAT, ENCODING=RLE",
+ "CREATE TIMESERIES root.test.d0.s1 WITH DATATYPE=TEXT, ENCODING=PLAIN",
+ "CREATE TIMESERIES root.test.d1.g0.s0 WITH DATATYPE=INT32, ENCODING=RLE"));
+
+ for (String sql : metaDataSqls) {
+ statement.execute(sql);
+ }
+
+ // load vehicle
+ boolean hasError = false;
+ try {
+ statement.execute("load \"" + vehicleTmpDir + "\"");
+ } catch (Exception e) {
+ hasError = true;
+ assertTrue(
+ e.getMessage()
+ .contains(
+ "because root.vehicle.d0.s0 is INT32 in the loading TsFile but is INT64 in IoTDB."));
+ }
+ assertTrue(hasError);
+
+ statement.execute("load \"" + vehicleTmpDir + "\" verify=false");
+ assertEquals(
+ 1,
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.vehicle"))
+ .getSequenceFileTreeSet()
+ .size());
+
+ // load test
+ hasError = false;
+ try {
+ statement.execute("load \"" + testTmpDir + "\"");
+ } catch (Exception e) {
+ hasError = true;
+ assertTrue(
+ e.getMessage()
+ .contains(
+ "because root.test.d0.s0 is INT32 in the loading TsFile but is FLOAT in IoTDB."));
+ }
+ assertTrue(hasError);
+
+ statement.execute("load \"" + testTmpDir + "\" verify=false");
+ assertEquals(
+ 2,
+ StorageEngine.getInstance()
+ .getProcessor(new PartialPath("root.test"))
+ .getSequenceFileTreeSet()
+ .size());
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test
public void removeTsFileTest() throws SQLException {
try (Connection connection =
DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
diff --git a/server/src/test/java/org/apache/iotdb/db/qp/physical/PhysicalPlanTest.java b/server/src/test/java/org/apache/iotdb/db/qp/physical/PhysicalPlanTest.java
index c6c0437..fe7f783 100644
--- a/server/src/test/java/org/apache/iotdb/db/qp/physical/PhysicalPlanTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/qp/physical/PhysicalPlanTest.java
@@ -1009,34 +1009,58 @@ public class PhysicalPlanTest {
OperateFilePlan plan = (OperateFilePlan) processor.parseSQLToPhysicalPlan(metadata);
assertEquals(
String.format(
- "OperateFilePlan{file=%s, targetDir=null, autoCreateSchema=true, sgLevel=1, operatorType=LOAD_FILES}",
+ "OperateFilePlan{file=%s, targetDir=null, autoCreateSchema=true, sgLevel=1, verify=true, "
+ + "operatorType=LOAD_FILES}",
filePath),
plan.toString());
- metadata = String.format("load \"%s\" true", filePath);
+ metadata = String.format("load \"%s\" autoregister=true", filePath);
processor = new Planner();
plan = (OperateFilePlan) processor.parseSQLToPhysicalPlan(metadata);
assertEquals(
String.format(
- "OperateFilePlan{file=%s, targetDir=null, autoCreateSchema=true, sgLevel=1, operatorType=LOAD_FILES}",
+ "OperateFilePlan{file=%s, targetDir=null, autoCreateSchema=true, sgLevel=1, verify=true, "
+ + "operatorType=LOAD_FILES}",
filePath),
plan.toString());
- metadata = String.format("load \"%s\" false", filePath);
+ metadata = String.format("load \"%s\" autoregister=false", filePath);
processor = new Planner();
plan = (OperateFilePlan) processor.parseSQLToPhysicalPlan(metadata);
assertEquals(
String.format(
- "OperateFilePlan{file=%s, targetDir=null, autoCreateSchema=false, sgLevel=1, operatorType=LOAD_FILES}",
+ "OperateFilePlan{file=%s, targetDir=null, autoCreateSchema=false, sgLevel=1, verify=true, "
+ + "operatorType=LOAD_FILES}",
filePath),
plan.toString());
- metadata = String.format("load \"%s\" true 3", filePath);
+ metadata = String.format("load \"%s\" autoregister=true,sglevel=3", filePath);
processor = new Planner();
plan = (OperateFilePlan) processor.parseSQLToPhysicalPlan(metadata);
assertEquals(
String.format(
- "OperateFilePlan{file=%s, targetDir=null, autoCreateSchema=true, sgLevel=3, operatorType=LOAD_FILES}",
+ "OperateFilePlan{file=%s, targetDir=null, autoCreateSchema=true, sgLevel=3, verify=true, "
+ + "operatorType=LOAD_FILES}",
+ filePath),
+ plan.toString());
+
+ metadata = String.format("load \"%s\" autoregister=true,sglevel=3,verify=false", filePath);
+ processor = new Planner();
+ plan = (OperateFilePlan) processor.parseSQLToPhysicalPlan(metadata);
+ assertEquals(
+ String.format(
+ "OperateFilePlan{file=%s, targetDir=null, autoCreateSchema=true, sgLevel=3, verify=false, "
+ + "operatorType=LOAD_FILES}",
+ filePath),
+ plan.toString());
+
+ metadata = String.format("load \"%s\" autoregister=true,sglevel=3,verify=true", filePath);
+ processor = new Planner();
+ plan = (OperateFilePlan) processor.parseSQLToPhysicalPlan(metadata);
+ assertEquals(
+ String.format(
+ "OperateFilePlan{file=%s, targetDir=null, autoCreateSchema=true, sgLevel=3, verify=true, "
+ + "operatorType=LOAD_FILES}",
filePath),
plan.toString());
}
@@ -1049,7 +1073,7 @@ public class PhysicalPlanTest {
OperateFilePlan plan = (OperateFilePlan) processor.parseSQLToPhysicalPlan(metadata);
assertEquals(
String.format(
- "OperateFilePlan{file=%s, targetDir=null, autoCreateSchema=false, sgLevel=0, operatorType=REMOVE_FILE}",
+ "OperateFilePlan{file=%s, targetDir=null, autoCreateSchema=false, sgLevel=0, verify=false, operatorType=REMOVE_FILE}",
filePath),
plan.toString());
}
@@ -1063,7 +1087,7 @@ public class PhysicalPlanTest {
OperateFilePlan plan = (OperateFilePlan) processor.parseSQLToPhysicalPlan(metadata);
assertEquals(
String.format(
- "OperateFilePlan{file=%s, targetDir=%s, autoCreateSchema=false, sgLevel=0, operatorType=MOVE_FILE}",
+ "OperateFilePlan{file=%s, targetDir=%s, autoCreateSchema=false, sgLevel=0, verify=false, operatorType=MOVE_FILE}",
filePath, targetDir),
plan.toString());
}
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/TsFileRewriteToolTest.java b/server/src/test/java/org/apache/iotdb/db/utils/TsFileRewriteToolTest.java
index 58126ef..9f3475b 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/TsFileRewriteToolTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/TsFileRewriteToolTest.java
@@ -173,7 +173,7 @@ public class TsFileRewriteToolTest {
deviceSensorsMap.put(DEVICE1, sensors);
createOneTsFile(deviceSensorsMap);
// try load the tsfile
- String sql = "load \"" + path + "\"" + " true";
+ String sql = "load \"" + path + "\"" + " autoregister=true";
try {
queryExecutor.processNonQuery(processor.parseSQLToPhysicalPlan(sql));
} catch (Exception e) {
@@ -189,7 +189,7 @@ public class TsFileRewriteToolTest {
deviceSensorsMap.put(DEVICE1, sensors);
createOneTsFileWithOnlyOnePage(deviceSensorsMap);
// try load the tsfile
- String sql = "load \"" + path + "\"" + " true";
+ String sql = "load \"" + path + "\"" + " autoregister=true";
try {
queryExecutor.processNonQuery(processor.parseSQLToPhysicalPlan(sql));
} catch (Exception e) {