You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ju...@apache.org on 2026/05/09 04:25:07 UTC
(iotdb) branch load-pri created (now c7f312e1bc0)
This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a change to branch load-pri
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at c7f312e1bc0 Load pri
This branch includes the following new commits:
new c7f312e1bc0 Load pri
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: Load pri
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a commit to branch load-pri
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c7f312e1bc0f998144897fd57f189c8caacb9bce
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Sat May 9 12:24:20 2026 +0800
Load pri
---
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 7 ++-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 18 ++++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 21 +++++++
.../protocol/legacy/loader/TsFileLoader.java | 2 +-
.../protocol/thrift/IoTDBDataNodeReceiver.java | 2 +-
.../plan/analyze/load/LoadTsFileAnalyzer.java | 29 ++++++++--
.../load/TreeSchemaAutoCreatorAndVerifier.java | 36 +++++++-----
.../relational/analyzer/StatementAnalyzer.java | 5 ++
.../relational/security/TableModelPrivilege.java | 6 ++
.../security/TreeAccessCheckVisitor.java | 11 +++-
.../plan/relational/sql/ast/LoadTsFile.java | 17 +++++-
.../plan/scheduler/load/LoadTsFileScheduler.java | 4 +-
.../plan/statement/crud/LoadTsFileStatement.java | 66 ++++++++++++++++++++--
.../load/active/ActiveLoadTsFileLoader.java | 3 +-
.../org/apache/iotdb/db/auth/TreeAccessTest.java | 37 ++++++++++++
.../relational/sql/parser/AuthorStatementTest.java | 4 ++
.../statement/crud/LoadTsFileStatementTest.java | 35 ++++++++++++
.../conf/iotdb-system.properties.template | 6 ++
.../iotdb/commons/auth/entity/PrivilegeType.java | 6 +-
.../org/apache/iotdb/commons/utils/AuthUtils.java | 5 ++
.../db/relational/grammar/sql/RelationalSql.g4 | 4 +-
21 files changed, 288 insertions(+), 36 deletions(-)
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
index 5c5cbe4a186..b131e403266 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
@@ -1129,6 +1129,7 @@ PRIVILEGE_VALUE
| EXTEND_TEMPLATE
| MANAGE_DATABASE
| MAINTAIN
+ | LOAD_TSFILE
;
READ_DATA
@@ -1187,6 +1188,10 @@ MAINTAIN
: M A I N T A I N
;
+LOAD_TSFILE
+ : L O A D '_' T S F I L E
+ ;
+
SECURITY
: S E C U R I T Y
;
@@ -1399,4 +1404,4 @@ fragment V: [vV];
fragment W: [wW];
fragment X: [xX];
fragment Y: [yY];
-fragment Z: [zZ];
\ No newline at end of file
+fragment Z: [zZ];
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 9b177cffcfa..8613bc318da 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -298,6 +298,8 @@ public class IoTDBConfig {
tierDataDirs[0][0] + File.separator + IoTDBConstant.LOAD_TSFILE_FOLDER_NAME
};
+ private String[] loadTsFileAllowedDirs = new String[0];
+
/** Strategy of multiple directories. */
private String multiDirStrategyClassName = null;
@@ -1355,6 +1357,9 @@ public class IoTDBConfig {
for (int i = 0; i < loadActiveListeningDirs.length; i++) {
loadActiveListeningDirs[i] = addDataHomeDir(loadActiveListeningDirs[i]);
}
+ for (int i = 0; i < loadTsFileAllowedDirs.length; i++) {
+ loadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]);
+ }
loadActiveListeningPipeDir = addDataHomeDir(loadActiveListeningPipeDir);
loadActiveListeningFailDir = addDataHomeDir(loadActiveListeningFailDir);
udfDir = addDataHomeDir(udfDir);
@@ -1560,6 +1565,19 @@ public class IoTDBConfig {
return this.loadTsFileDirs;
}
+ public String[] getLoadTsFileAllowedDirs() {
+ return this.loadTsFileAllowedDirs.length == 0
+ ? getLoadTsFileDirs()
+ : this.loadTsFileAllowedDirs;
+ }
+
+ public void setLoadTsFileAllowedDirs(String[] loadTsFileAllowedDirs) {
+ for (int i = 0; i < loadTsFileAllowedDirs.length; i++) {
+ loadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]);
+ }
+ this.loadTsFileAllowedDirs = loadTsFileAllowedDirs;
+ }
+
public void formulateLoadTsFileDirs(String[][] tierDataDirs) {
if (tierDataDirs.length < 1) {
logger.warn("No data directory is set. loadTsFileDirs is kept as the default value.");
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 7193fd27c58..beceafc6494 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -2456,6 +2456,15 @@ public class IoTDBDescriptor {
"load_write_throughput_bytes_per_second",
String.valueOf(conf.getLoadWriteThroughputBytesPerSecond()))));
+ conf.setLoadTsFileAllowedDirs(
+ Arrays.stream(
+ properties
+ .getProperty("load_tsfile_allowed_dirs", "")
+ .trim()
+ .split(","))
+ .filter(dir -> !dir.isEmpty())
+ .toArray(String[]::new));
+
conf.setLoadTabletConversionThresholdBytes(
Long.parseLong(
properties.getProperty(
@@ -2573,6 +2582,18 @@ public class IoTDBDescriptor {
ConfigurationFileUtils.getConfigurationDefaultValue(
"load_write_throughput_bytes_per_second"))));
+ conf.setLoadTsFileAllowedDirs(
+ Arrays.stream(
+ properties
+ .getProperty(
+ "load_tsfile_allowed_dirs",
+ ConfigurationFileUtils.getConfigurationDefaultValue(
+ "load_tsfile_allowed_dirs"))
+ .trim()
+ .split(","))
+ .filter(dir -> !dir.isEmpty())
+ .toArray(String[]::new));
+
conf.setLoadActiveListeningEnable(
Boolean.parseBoolean(
properties.getProperty(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java
index 1c95b574b54..92139942b0c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java
@@ -55,7 +55,7 @@ public class TsFileLoader implements ILoader {
@Override
public void load() {
try {
- LoadTsFileStatement statement = new LoadTsFileStatement(tsFile.getAbsolutePath());
+ LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath());
statement.setDeleteAfterLoad(true);
statement.setConvertOnTypeMismatch(true);
statement.setDatabaseLevel(parseSgLevel());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
index c10fdbc4f67..dd6e684031c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
@@ -585,7 +585,7 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver {
private TSStatus loadTsFileSync(final String dataBaseName, final String fileAbsolutePath)
throws FileNotFoundException {
- final LoadTsFileStatement statement = new LoadTsFileStatement(fileAbsolutePath);
+ final LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(fileAbsolutePath);
statement.setDeleteAfterLoad(true);
statement.setConvertOnTypeMismatch(true);
statement.setVerifySchema(validateTsFile.get());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
index e1e6d597191..0538ecbc576 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
@@ -53,6 +53,7 @@ import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.encrypt.EncryptParameter;
import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.external.commons.io.FileUtils;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.TableSchema;
@@ -443,14 +444,15 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
isTableModelTsFile.get(i)
? loadTsFileDataTypeConverter
.convertForTableModel(
- new LoadTsFile(null, tsFiles.get(i).getPath(), Collections.emptyMap())
+ LoadTsFile.createUnchecked(
+ null, tsFiles.get(i).getPath(), Collections.emptyMap())
.setDatabase(databaseForTableData)
.setDeleteAfterLoad(isDeleteAfterLoad)
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
.orElse(null)
: loadTsFileDataTypeConverter
.convertForTreeModel(
- new LoadTsFileStatement(tsFiles.get(i).getPath())
+ LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath())
.setDeleteAfterLoad(isDeleteAfterLoad)
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
.orElse(null);
@@ -505,6 +507,8 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
if (isAutoCreateSchemaOrVerifySchemaEnabled) {
getOrCreateTreeSchemaVerifier().autoCreateAndVerify(reader, device2TimeseriesMetadata);
+ } else {
+ checkTreeWritePermission(device2TimeseriesMetadata);
}
// TODO: how to get the correct write point count when
@@ -522,6 +526,22 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
addWritePointCount(writePointCount);
}
+ private void checkTreeWritePermission(
+ final Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadataList)
+ throws AuthException {
+ for (final Map.Entry<IDeviceID, List<TimeseriesMetadata>> entry :
+ device2TimeseriesMetadataList.entrySet()) {
+ final IDeviceID device = entry.getKey();
+ for (final TimeseriesMetadata timeseriesMetadata : entry.getValue()) {
+ if (!TSDataType.VECTOR.equals(timeseriesMetadata.getTsDataType())
+ && !timeseriesMetadata.getMeasurementId().isEmpty()) {
+ TreeSchemaAutoCreatorAndVerifier.checkWriteDataPermission(
+ this, device, timeseriesMetadata);
+ }
+ }
+ }
+ }
+
private void doAnalyzeSingleTableFile(
final File tsFile,
final TsFileSequenceReader reader,
@@ -711,14 +731,15 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
isTableModelTsFile.get(i)
? loadTsFileDataTypeConverter
.convertForTableModel(
- new LoadTsFile(null, tsFiles.get(i).getPath(), Collections.emptyMap())
+ LoadTsFile.createUnchecked(
+ null, tsFiles.get(i).getPath(), Collections.emptyMap())
.setDatabase(databaseForTableData)
.setDeleteAfterLoad(isDeleteAfterLoad)
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
.orElse(null)
: loadTsFileDataTypeConverter
.convertForTreeModel(
- new LoadTsFileStatement(tsFiles.get(i).getPath())
+ LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath())
.setDeleteAfterLoad(isDeleteAfterLoad)
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
.orElse(null);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java
index de56a8603c5..d8fbb9c80d7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java
@@ -149,21 +149,7 @@ public class TreeSchemaAutoCreatorAndVerifier {
// not a timeseries, skip
} else {
- // check WRITE_DATA permission of timeseries
- long startTime = System.nanoTime();
- try {
- UserEntity userEntity = loadTsFileAnalyzer.context.getSession().getUserEntity();
- TSStatus status =
- AuthorityChecker.getAccessControl()
- .checkFullPathWriteDataPermission(
- userEntity, device, timeseriesMetadata.getMeasurementId());
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- throw new AuthException(
- TSStatusCode.representOf(status.getCode()), status.getMessage());
- }
- } finally {
- PerformanceOverviewMetrics.getInstance().recordAuthCost(System.nanoTime() - startTime);
- }
+ checkWriteDataPermission(loadTsFileAnalyzer, device, timeseriesMetadata);
final Pair<CompressionType, TSEncoding> compressionEncodingPair =
reader.readTimeseriesCompressionTypeAndEncoding(timeseriesMetadata);
schemaCache.addTimeSeries(
@@ -187,6 +173,26 @@ public class TreeSchemaAutoCreatorAndVerifier {
}
}
+ static void checkWriteDataPermission(
+ final LoadTsFileAnalyzer loadTsFileAnalyzer,
+ final IDeviceID device,
+ final TimeseriesMetadata timeseriesMetadata)
+ throws AuthException {
+ final long startTime = System.nanoTime();
+ try {
+ final UserEntity userEntity = loadTsFileAnalyzer.context.getSession().getUserEntity();
+ final TSStatus status =
+ AuthorityChecker.getAccessControl()
+ .checkFullPathWriteDataPermission(
+ userEntity, device, timeseriesMetadata.getMeasurementId());
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new AuthException(TSStatusCode.representOf(status.getCode()), status.getMessage());
+ }
+ } finally {
+ PerformanceOverviewMetrics.getInstance().recordAuthCost(System.nanoTime() - startTime);
+ }
+ }
+
/**
* This can only be invoked after all timeseries in the current tsfile have been processed.
* Otherwise, the isAligned status may be wrong.
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
index 6d3326dd402..87803d70d79 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.queryengine.plan.relational.analyzer;
import org.apache.iotdb.calc.plan.relational.metadata.CommonMetadataUtils;
+import org.apache.iotdb.commons.auth.entity.PrivilegeType;
import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.commons.exception.SemanticException;
import org.apache.iotdb.commons.queryengine.common.SessionInfo;
@@ -830,6 +831,10 @@ public class StatementAnalyzer {
@Override
public Scope visitLoadTsFile(final LoadTsFile node, final Optional<Scope> scope) {
queryContext.setQueryType(QueryType.OTHER);
+ accessControl.checkMissingPrivileges(
+ sessionContext.getUserName(),
+ Collections.singletonList(PrivilegeType.LOAD_TSFILE),
+ queryContext);
try (final LoadTsFileAnalyzer loadTsFileAnalyzer =
new LoadTsFileAnalyzer(node, node.isGeneratedByPipe(), queryContext)) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TableModelPrivilege.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TableModelPrivilege.java
index d06a08ef026..3fc99e55f41 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TableModelPrivilege.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TableModelPrivilege.java
@@ -32,6 +32,7 @@ public enum TableModelPrivilege {
SYSTEM,
SECURITY,
AUDIT,
+ LOAD_TSFILE,
// scope privilege
CREATE,
@@ -65,6 +66,8 @@ public enum TableModelPrivilege {
return PrivilegeType.SECURITY;
case AUDIT:
return PrivilegeType.AUDIT;
+ case LOAD_TSFILE:
+ return PrivilegeType.LOAD_TSFILE;
default:
throw new IllegalStateException("Unexpected value:" + this);
}
@@ -94,6 +97,8 @@ public enum TableModelPrivilege {
return TableModelPrivilege.SECURITY;
case AUDIT:
return TableModelPrivilege.AUDIT;
+ case LOAD_TSFILE:
+ return TableModelPrivilege.LOAD_TSFILE;
default:
throw new IllegalStateException("Unexpected value:" + privilegeType);
}
@@ -109,6 +114,7 @@ public enum TableModelPrivilege {
return AuditLogOperation.QUERY;
case INSERT:
case DELETE:
+ case LOAD_TSFILE:
return AuditLogOperation.DML;
case MANAGE_ROLE:
case MANAGE_USER:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java
index 7b4655c79a8..e525a5e061d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java
@@ -169,6 +169,7 @@ import org.apache.iotdb.rpc.TSStatusCode;
import com.google.common.collect.ImmutableList;
+import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -1149,8 +1150,14 @@ public class TreeAccessCheckVisitor extends StatementVisitor<TSStatus, TreeAcces
@Override
public TSStatus visitLoadFile(LoadTsFileStatement statement, TreeAccessCheckContext context) {
- // no need to check here, it will be checked in process phase
- return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ return checkGlobalAuth(
+ context.setAuditLogOperation(AuditLogOperation.DML),
+ PrivilegeType.LOAD_TSFILE,
+ () ->
+ statement.getTsFiles().stream()
+ .map(File::getPath)
+ .collect(Collectors.toList())
+ .toString());
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java
index 03de9a9ccb1..dc1a549f636 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java
@@ -72,6 +72,19 @@ public class LoadTsFile extends Statement {
private boolean needDecode4TimeColumn;
public LoadTsFile(NodeLocation location, String filePath, Map<String, String> loadAttributes) {
+ this(location, filePath, loadAttributes, true);
+ }
+
+ public static LoadTsFile createUnchecked(
+ NodeLocation location, String filePath, Map<String, String> loadAttributes) {
+ return new LoadTsFile(location, filePath, loadAttributes, false);
+ }
+
+ private LoadTsFile(
+ NodeLocation location,
+ String filePath,
+ Map<String, String> loadAttributes,
+ boolean validateSourcePath) {
super(location);
this.filePath = requireNonNull(filePath, "filePath is null");
@@ -89,7 +102,7 @@ public class LoadTsFile extends Statement {
try {
this.tsFiles =
org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement.processTsFile(
- new File(filePath));
+ new File(filePath), validateSourcePath);
this.resources = new ArrayList<>();
this.writePointCountList = new ArrayList<>();
this.isTableModel = new ArrayList<>(Collections.nCopies(this.tsFiles.size(), true));
@@ -283,7 +296,7 @@ public class LoadTsFile extends Statement {
final Map<String, String> properties = this.loadAttributes;
final LoadTsFile subStatement =
- new LoadTsFile(getLocation().orElse(null), filePath, properties);
+ LoadTsFile.createUnchecked(getLocation().orElse(null), filePath, properties);
// Copy all configuration properties
subStatement.databaseLevel = this.databaseLevel;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
index bc8446fd3da..6671dad3548 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
@@ -586,14 +586,14 @@ public class LoadTsFileScheduler implements IScheduler {
failedNode.isTableModel()
? loadTsFileDataTypeConverter
.convertForTableModel(
- new LoadTsFile(null, filePath, Collections.emptyMap())
+ LoadTsFile.createUnchecked(null, filePath, Collections.emptyMap())
.setDatabase(failedNode.getDatabase())
.setDeleteAfterLoad(failedNode.isDeleteAfterLoad())
.setConvertOnTypeMismatch(true))
.orElse(null)
: loadTsFileDataTypeConverter
.convertForTreeModel(
- new LoadTsFileStatement(filePath)
+ LoadTsFileStatement.createUnchecked(filePath)
.setDeleteAfterLoad(failedNode.isDeleteAfterLoad())
.setConvertOnTypeMismatch(true))
.orElse(null);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
index 512a62fdae7..a846c7feb43 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
@@ -35,7 +35,9 @@ import org.apache.tsfile.common.constant.TsFileConstant;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -71,6 +73,15 @@ public class LoadTsFileStatement extends Statement {
private boolean needDecode4TimeColumn;
public LoadTsFileStatement(String filePath) throws FileNotFoundException {
+ this(filePath, true);
+ }
+
+ public static LoadTsFileStatement createUnchecked(String filePath) throws FileNotFoundException {
+ return new LoadTsFileStatement(filePath, false);
+ }
+
+ private LoadTsFileStatement(String filePath, boolean validateSourcePath)
+ throws FileNotFoundException {
this.file = new File(filePath).getAbsoluteFile();
this.databaseLevel = IoTDBDescriptor.getInstance().getConfig().getDefaultDatabaseLevel();
this.verifySchema = true;
@@ -80,7 +91,7 @@ public class LoadTsFileStatement extends Statement {
IoTDBDescriptor.getInstance().getConfig().getLoadTabletConversionThresholdBytes();
this.autoCreateDatabase = IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled();
- this.tsFiles = processTsFile(file);
+ this.tsFiles = processTsFile(file, validateSourcePath);
this.resources = new ArrayList<>();
this.writePointCountList = new ArrayList<>();
this.isTableModel = new ArrayList<>(Collections.nCopies(this.tsFiles.size(), false));
@@ -88,6 +99,15 @@ public class LoadTsFileStatement extends Statement {
}
public static List<File> processTsFile(final File file) throws FileNotFoundException {
+ return processTsFile(file, true);
+ }
+
+ public static List<File> processTsFile(final File file, final boolean validateSourcePath)
+ throws FileNotFoundException {
+ if (validateSourcePath) {
+ validateLoadSourcePath(file);
+ }
+
final List<File> tsFiles = new ArrayList<>();
if (file.isFile()) {
tsFiles.add(file);
@@ -98,7 +118,7 @@ public class LoadTsFileStatement extends Statement {
"Can not find %s on this machine, notice that load can only handle files on this machine.",
file.getPath()));
}
- tsFiles.addAll(findAllTsFile(file));
+ tsFiles.addAll(findAllTsFile(file, validateSourcePath));
}
sortTsFiles(tsFiles);
return tsFiles;
@@ -120,7 +140,8 @@ public class LoadTsFileStatement extends Statement {
this.statementType = StatementType.MULTI_BATCH_INSERT;
}
- private static List<File> findAllTsFile(File file) {
+ private static List<File> findAllTsFile(File file, boolean validateSourcePath)
+ throws FileNotFoundException {
final File[] files = file.listFiles();
if (files == null) {
return Collections.emptyList();
@@ -128,15 +149,50 @@ public class LoadTsFileStatement extends Statement {
final List<File> tsFiles = new ArrayList<>();
for (File nowFile : files) {
+ if (validateSourcePath) {
+ validateLoadSourcePath(nowFile);
+ }
if (nowFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) {
tsFiles.add(nowFile);
} else if (nowFile.isDirectory()) {
- tsFiles.addAll(findAllTsFile(nowFile));
+ tsFiles.addAll(findAllTsFile(nowFile, validateSourcePath));
}
}
return tsFiles;
}
+ public static void validateLoadSourcePath(final String filePath) throws FileNotFoundException {
+ validateLoadSourcePath(new File(filePath));
+ }
+
+ private static void validateLoadSourcePath(final File file) throws FileNotFoundException {
+ final Path sourcePath = canonicalPath(file);
+ final String[] allowedDirs =
+ IoTDBDescriptor.getInstance().getConfig().getLoadTsFileAllowedDirs();
+
+ for (final String allowedDir : allowedDirs) {
+ if (sourcePath.startsWith(canonicalPath(new File(allowedDir)))) {
+ return;
+ }
+ }
+
+ throw new FileNotFoundException(
+ String.format(
+ "Load TsFile source path %s is outside allowed directories %s.",
+ sourcePath, Arrays.toString(allowedDirs)));
+ }
+
+ private static Path canonicalPath(final File file) throws FileNotFoundException {
+ try {
+ return file.getCanonicalFile().toPath();
+ } catch (final IOException e) {
+ throw new FileNotFoundException(
+ String.format(
+ "Failed to resolve canonical path for Load TsFile source %s: %s",
+ file.getPath(), e.getMessage()));
+ }
+ }
+
private static void sortTsFiles(List<File> files) {
files.sort(
(o1, o2) -> {
@@ -389,7 +445,7 @@ public class LoadTsFileStatement extends Statement {
loadAttributes.put(PIPE_GENERATED_KEY, String.valueOf(true));
}
- return new LoadTsFile(null, file.getAbsolutePath(), loadAttributes);
+ return LoadTsFile.createUnchecked(null, file.getAbsolutePath(), loadAttributes);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java
index d0be2ead5cb..67fa0fff300 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java
@@ -219,7 +219,8 @@ public class ActiveLoadTsFileLoader {
final ActiveLoadPendingQueue.ActiveLoadEntry entry, final IClientSession session)
throws FileNotFoundException {
final File tsFile = new File(entry.getFile());
- final LoadTsFileStatement statement = new LoadTsFileStatement(tsFile.getAbsolutePath());
+ final LoadTsFileStatement statement =
+ LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath());
final List<File> files = statement.getTsFiles();
statement.setDeleteAfterLoad(true);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/TreeAccessTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/TreeAccessTest.java
index af8746fae3e..6eff04b0e47 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/TreeAccessTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/TreeAccessTest.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.auth.entity.User;
import org.apache.iotdb.db.queryengine.plan.relational.security.TreeAccessCheckContext;
import org.apache.iotdb.db.queryengine.plan.relational.security.TreeAccessCheckVisitor;
import org.apache.iotdb.db.queryengine.plan.statement.AuthorType;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.AuthorStatement;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -33,6 +34,9 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
public class TreeAccessTest {
@Before
@@ -81,4 +85,37 @@ public class TreeAccessTest {
.visitAuthor(authorStatement, new TreeAccessCheckContext(10000L, "user1", ""))
.getCode());
}
+
+ @Test
+ public void testLoadTsFileRequiresLoadTsFilePrivilege() throws Exception {
+ final User mockUser = Mockito.mock(User.class);
+ Mockito.when(mockUser.getName()).thenReturn("loadUser");
+ Mockito.when(mockUser.getUserId()).thenReturn(10002L);
+ Mockito.when(mockUser.checkSysPrivilege(PrivilegeType.LOAD_TSFILE)).thenReturn(false);
+ Mockito.when(mockUser.checkSysPrivilege(PrivilegeType.SYSTEM)).thenReturn(false);
+ AuthorityChecker.getAuthorityFetcher()
+ .getAuthorCache()
+ .putUserCache(mockUser.getName(), mockUser);
+
+ final Path tsFile = Files.createTempFile("load-auth", ".tsfile");
+ try {
+ final LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(tsFile.toString());
+ final TreeAccessCheckVisitor treeAccessCheckVisitor = new TreeAccessCheckVisitor();
+
+ Assert.assertEquals(
+ TSStatusCode.NO_PERMISSION.getStatusCode(),
+ treeAccessCheckVisitor
+ .visitLoadFile(statement, new TreeAccessCheckContext(10002L, "loadUser", ""))
+ .getCode());
+
+ Mockito.when(mockUser.checkSysPrivilege(PrivilegeType.LOAD_TSFILE)).thenReturn(true);
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ treeAccessCheckVisitor
+ .visitLoadFile(statement, new TreeAccessCheckContext(10002L, "loadUser", ""))
+ .getCode());
+ } finally {
+ Files.deleteIfExists(tsFile);
+ }
+ }
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AuthorStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AuthorStatementTest.java
index 509c529b36d..bb43f35f316 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AuthorStatementTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AuthorStatementTest.java
@@ -238,6 +238,8 @@ public class AuthorStatementTest {
checkGrantPrivileges(true, Collections.singletonList(PrivilegeType.SECURITY), "", false, false);
checkGrantPrivileges(
true, Arrays.asList(PrivilegeType.SYSTEM, PrivilegeType.SECURITY), "", true, false);
+ checkGrantPrivileges(
+ true, Collections.singletonList(PrivilegeType.LOAD_TSFILE), "", true, false);
checkGrantPrivileges(
true, Collections.singletonList(PrivilegeType.SECURITY), "ANY", true, true);
@@ -301,6 +303,8 @@ public class AuthorStatementTest {
true, Arrays.asList(PrivilegeType.SYSTEM, PrivilegeType.SECURITY), "", false, false);
checkRevokePrivileges(
true, Arrays.asList(PrivilegeType.SYSTEM, PrivilegeType.SECURITY), "", true, false);
+ checkRevokePrivileges(
+ true, Collections.singletonList(PrivilegeType.LOAD_TSFILE), "", false, false);
// Illegal privileges combination
checkRevokePrivileges(
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java
index 941794bb074..25644e97c71 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.junit.Assert;
import org.junit.Test;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -38,10 +39,12 @@ public class LoadTsFileStatementTest {
public void testSubStatementsKeepDatabase() throws Exception {
final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
final int originalBatchSize = config.getLoadTsFileSubStatementBatchSize();
+ final String[] originalAllowedDirs = config.getLoadTsFileAllowedDirs().clone();
final Path tempDir = Files.createTempDirectory("load-tsfile-sub-statements");
try {
config.setLoadTsFileSubStatementBatchSize(1);
+ config.setLoadTsFileAllowedDirs(new String[] {tempDir.toString()});
Files.createFile(tempDir.resolve("a.tsfile"));
Files.createFile(tempDir.resolve("b.tsfile"));
@@ -54,10 +57,42 @@ public class LoadTsFileStatementTest {
subStatement -> Assert.assertEquals("test_db", subStatement.getDatabase()));
} finally {
config.setLoadTsFileSubStatementBatchSize(originalBatchSize);
+ config.setLoadTsFileAllowedDirs(originalAllowedDirs);
deleteRecursively(tempDir);
}
}
+ @Test
+ public void testLoadSourcePathMustBeInAllowedDirs() throws Exception {
+ final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ final String[] originalAllowedDirs = config.getLoadTsFileAllowedDirs().clone();
+ final Path allowedDir = Files.createTempDirectory("load-tsfile-allowed");
+ final Path deniedDir = Files.createTempDirectory("load-tsfile-denied");
+
+ try {
+ config.setLoadTsFileAllowedDirs(new String[] {allowedDir.toString()});
+ final Path deniedTsFile = Files.createFile(deniedDir.resolve("denied.tsfile"));
+ final Path traversalTsFile =
+ allowedDir.resolve("..").resolve(deniedDir.getFileName()).resolve("denied.tsfile");
+
+ assertLoadSourcePathRejected(deniedTsFile);
+ assertLoadSourcePathRejected(traversalTsFile);
+ } finally {
+ config.setLoadTsFileAllowedDirs(originalAllowedDirs);
+ deleteRecursively(allowedDir);
+ deleteRecursively(deniedDir);
+ }
+ }
+
+ private static void assertLoadSourcePathRejected(final Path sourcePath) {
+ try {
+ new LoadTsFileStatement(sourcePath.toString());
+ Assert.fail("Expected disallowed LOAD TSFILE source path to be rejected.");
+ } catch (final FileNotFoundException e) {
+ Assert.assertTrue(e.getMessage().contains("outside allowed directories"));
+ }
+ }
+
private static void deleteRecursively(final Path path) throws IOException {
if (path == null || !Files.exists(path)) {
return;
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 69bcd275250..7606909a039 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -2182,6 +2182,12 @@ load_clean_up_task_execution_delay_time_seconds=1800
# Datatype: int
load_write_throughput_bytes_per_second=-1
+# Comma-separated list of directories from which user-issued LOAD TSFILE statements can read.
+# If empty, IoTDB only permits LOAD sources under the internal load TsFile directories.
+# effectiveMode: hot_reload
+# Datatype: String
+load_tsfile_allowed_dirs=
+
# Whether to enable the active listening mode for tsfile loading.
# effectiveMode: hot_reload
# Datatype: Boolean
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/PrivilegeType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/PrivilegeType.java
index 29dc5301a84..5d0b4359840 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/PrivilegeType.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/PrivilegeType.java
@@ -54,7 +54,8 @@ public enum PrivilegeType {
SYSTEM(PrivilegeModelType.SYSTEM),
SECURITY(PrivilegeModelType.SYSTEM),
- AUDIT(PrivilegeModelType.SYSTEM);
+ AUDIT(PrivilegeModelType.SYSTEM),
+ LOAD_TSFILE(PrivilegeModelType.SYSTEM);
private final PrivilegeModelType modelType;
@@ -134,6 +135,7 @@ public enum PrivilegeType {
switch (this) {
case MANAGE_USER:
case MANAGE_ROLE:
+ case LOAD_TSFILE:
case SYSTEM:
case SECURITY:
case AUDIT:
@@ -160,6 +162,7 @@ public enum PrivilegeType {
case USE_PIPE:
case MANAGE_DATABASE:
case EXTEND_TEMPLATE:
+ case LOAD_TSFILE:
return Arrays.asList(this, PrivilegeType.SYSTEM);
default:
return Collections.singletonList(this);
@@ -208,6 +211,7 @@ public enum PrivilegeType {
case WRITE_SCHEMA:
case INSERT:
case DELETE:
+ case LOAD_TSFILE:
return AuditLogOperation.DML;
case MANAGE_USER:
case MANAGE_ROLE:
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/AuthUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/AuthUtils.java
index 1aae1909ef2..f6c636251e7 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/AuthUtils.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/AuthUtils.java
@@ -568,6 +568,8 @@ public class AuthUtils {
return PrivilegeType.SECURITY;
case 12:
return PrivilegeType.AUDIT;
+ case 13:
+ return PrivilegeType.LOAD_TSFILE;
default:
// Not reach here.
LOGGER.warn("Not support position");
@@ -603,6 +605,8 @@ public class AuthUtils {
return 11;
case AUDIT:
return 12;
+ case LOAD_TSFILE:
+ return 13;
default:
return -1;
}
@@ -621,6 +625,7 @@ public class AuthUtils {
case USE_PIPE:
case MANAGE_DATABASE:
case EXTEND_TEMPLATE:
+ case LOAD_TSFILE:
return Arrays.asList(priv, PrivilegeType.SYSTEM);
default:
return Collections.singletonList(priv);
diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
index ca766464686..8e94f92ed69 100644
--- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
+++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
@@ -829,6 +829,7 @@ systemPrivilege
| MANAGE_ROLE
| SYSTEM
| SECURITY
+ | LOAD_TSFILE
;
objectPrivilege
@@ -1493,7 +1494,7 @@ nonReserved
| INDEX | INDEXES | IF | IGNORE | IMMEDIATE | INCLUDING | INITIAL | INPUT | INTERVAL | INVOKER | IO | ITERATE | ISOLATION
| JSON
| KEEP | KEY | KEYS | KILL
- | LANGUAGE | LAST | LATERAL | LEADING | LEAVE | LEVEL | LIMIT | LINEAR | LOAD | LOCAL | LOGICAL | LOOP
+ | LANGUAGE | LAST | LATERAL | LEADING | LEAVE | LEVEL | LIMIT | LINEAR | LOAD | LOAD_TSFILE | LOCAL | LOGICAL | LOOP
| MANAGE_ROLE | MANAGE_USER | MAP | MATCH | MATCHED | MATCHES | MATCH_RECOGNIZE | MATERIALIZED | MEASURES | MEMORY_THRESHOLD | METHOD | MERGE | MICROSECOND | MIGRATE | MILLISECOND | MINUTE | MODEL | MODELS | MODIFY | MONTH
| NANOSECOND | NESTED | NEXT | NFC | NFD | NFKC | NFKD | NO | NODEID | NONE | NULLIF | NULLS
| OBJECT | OF | OFFSET | OMIT | ONE | ONLY | OPTION | ORDINALITY | OUTPUT | OVER | OVERFLOW
@@ -1700,6 +1701,7 @@ LINEAR: 'LINEAR';
LIST: 'LIST';
LISTAGG: 'LISTAGG';
LOAD: 'LOAD';
+LOAD_TSFILE: 'LOAD_TSFILE';
LOADED: 'LOADED';
LOCAL: 'LOCAL';
LOCALTIME: 'LOCALTIME';