You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ju...@apache.org on 2026/05/09 04:25:08 UTC
(iotdb) 01/01: Load pri
This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a commit to branch load-pri
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c7f312e1bc0f998144897fd57f189c8caacb9bce
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Sat May 9 12:24:20 2026 +0800
Load pri
---
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 7 ++-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 18 ++++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 21 +++++++
.../protocol/legacy/loader/TsFileLoader.java | 2 +-
.../protocol/thrift/IoTDBDataNodeReceiver.java | 2 +-
.../plan/analyze/load/LoadTsFileAnalyzer.java | 29 ++++++++--
.../load/TreeSchemaAutoCreatorAndVerifier.java | 36 +++++++-----
.../relational/analyzer/StatementAnalyzer.java | 5 ++
.../relational/security/TableModelPrivilege.java | 6 ++
.../security/TreeAccessCheckVisitor.java | 11 +++-
.../plan/relational/sql/ast/LoadTsFile.java | 17 +++++-
.../plan/scheduler/load/LoadTsFileScheduler.java | 4 +-
.../plan/statement/crud/LoadTsFileStatement.java | 66 ++++++++++++++++++++--
.../load/active/ActiveLoadTsFileLoader.java | 3 +-
.../org/apache/iotdb/db/auth/TreeAccessTest.java | 37 ++++++++++++
.../relational/sql/parser/AuthorStatementTest.java | 4 ++
.../statement/crud/LoadTsFileStatementTest.java | 35 ++++++++++++
.../conf/iotdb-system.properties.template | 6 ++
.../iotdb/commons/auth/entity/PrivilegeType.java | 6 +-
.../org/apache/iotdb/commons/utils/AuthUtils.java | 5 ++
.../db/relational/grammar/sql/RelationalSql.g4 | 4 +-
21 files changed, 288 insertions(+), 36 deletions(-)
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
index 5c5cbe4a186..b131e403266 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
@@ -1129,6 +1129,7 @@ PRIVILEGE_VALUE
| EXTEND_TEMPLATE
| MANAGE_DATABASE
| MAINTAIN
+ | LOAD_TSFILE
;
READ_DATA
@@ -1187,6 +1188,10 @@ MAINTAIN
: M A I N T A I N
;
+LOAD_TSFILE
+ : L O A D '_' T S F I L E
+ ;
+
SECURITY
: S E C U R I T Y
;
@@ -1399,4 +1404,4 @@ fragment V: [vV];
fragment W: [wW];
fragment X: [xX];
fragment Y: [yY];
-fragment Z: [zZ];
\ No newline at end of file
+fragment Z: [zZ];
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 9b177cffcfa..8613bc318da 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -298,6 +298,8 @@ public class IoTDBConfig {
tierDataDirs[0][0] + File.separator + IoTDBConstant.LOAD_TSFILE_FOLDER_NAME
};
+ private String[] loadTsFileAllowedDirs = new String[0];
+
/** Strategy of multiple directories. */
private String multiDirStrategyClassName = null;
@@ -1355,6 +1357,9 @@ public class IoTDBConfig {
for (int i = 0; i < loadActiveListeningDirs.length; i++) {
loadActiveListeningDirs[i] = addDataHomeDir(loadActiveListeningDirs[i]);
}
+ for (int i = 0; i < loadTsFileAllowedDirs.length; i++) {
+ loadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]);
+ }
loadActiveListeningPipeDir = addDataHomeDir(loadActiveListeningPipeDir);
loadActiveListeningFailDir = addDataHomeDir(loadActiveListeningFailDir);
udfDir = addDataHomeDir(udfDir);
@@ -1560,6 +1565,19 @@ public class IoTDBConfig {
return this.loadTsFileDirs;
}
+ public String[] getLoadTsFileAllowedDirs() {
+ return this.loadTsFileAllowedDirs.length == 0
+ ? getLoadTsFileDirs()
+ : this.loadTsFileAllowedDirs;
+ }
+
+ public void setLoadTsFileAllowedDirs(String[] loadTsFileAllowedDirs) {
+ for (int i = 0; i < loadTsFileAllowedDirs.length; i++) {
+ loadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]);
+ }
+ this.loadTsFileAllowedDirs = loadTsFileAllowedDirs;
+ }
+
public void formulateLoadTsFileDirs(String[][] tierDataDirs) {
if (tierDataDirs.length < 1) {
logger.warn("No data directory is set. loadTsFileDirs is kept as the default value.");
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 7193fd27c58..beceafc6494 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -2456,6 +2456,15 @@ public class IoTDBDescriptor {
"load_write_throughput_bytes_per_second",
String.valueOf(conf.getLoadWriteThroughputBytesPerSecond()))));
+ conf.setLoadTsFileAllowedDirs(
+ Arrays.stream(
+ properties
+ .getProperty("load_tsfile_allowed_dirs", "")
+ .trim()
+ .split(","))
+ .filter(dir -> !dir.isEmpty())
+ .toArray(String[]::new));
+
conf.setLoadTabletConversionThresholdBytes(
Long.parseLong(
properties.getProperty(
@@ -2573,6 +2582,18 @@ public class IoTDBDescriptor {
ConfigurationFileUtils.getConfigurationDefaultValue(
"load_write_throughput_bytes_per_second"))));
+ conf.setLoadTsFileAllowedDirs(
+ Arrays.stream(
+ properties
+ .getProperty(
+ "load_tsfile_allowed_dirs",
+ ConfigurationFileUtils.getConfigurationDefaultValue(
+ "load_tsfile_allowed_dirs"))
+ .trim()
+ .split(","))
+ .filter(dir -> !dir.isEmpty())
+ .toArray(String[]::new));
+
conf.setLoadActiveListeningEnable(
Boolean.parseBoolean(
properties.getProperty(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java
index 1c95b574b54..92139942b0c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java
@@ -55,7 +55,7 @@ public class TsFileLoader implements ILoader {
@Override
public void load() {
try {
- LoadTsFileStatement statement = new LoadTsFileStatement(tsFile.getAbsolutePath());
+ LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath());
statement.setDeleteAfterLoad(true);
statement.setConvertOnTypeMismatch(true);
statement.setDatabaseLevel(parseSgLevel());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
index c10fdbc4f67..dd6e684031c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java
@@ -585,7 +585,7 @@ public class IoTDBDataNodeReceiver extends IoTDBFileReceiver {
private TSStatus loadTsFileSync(final String dataBaseName, final String fileAbsolutePath)
throws FileNotFoundException {
- final LoadTsFileStatement statement = new LoadTsFileStatement(fileAbsolutePath);
+ final LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(fileAbsolutePath);
statement.setDeleteAfterLoad(true);
statement.setConvertOnTypeMismatch(true);
statement.setVerifySchema(validateTsFile.get());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
index e1e6d597191..0538ecbc576 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java
@@ -53,6 +53,7 @@ import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.encrypt.EncryptParameter;
import org.apache.tsfile.encrypt.EncryptUtils;
+import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.external.commons.io.FileUtils;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.TableSchema;
@@ -443,14 +444,15 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
isTableModelTsFile.get(i)
? loadTsFileDataTypeConverter
.convertForTableModel(
- new LoadTsFile(null, tsFiles.get(i).getPath(), Collections.emptyMap())
+ LoadTsFile.createUnchecked(
+ null, tsFiles.get(i).getPath(), Collections.emptyMap())
.setDatabase(databaseForTableData)
.setDeleteAfterLoad(isDeleteAfterLoad)
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
.orElse(null)
: loadTsFileDataTypeConverter
.convertForTreeModel(
- new LoadTsFileStatement(tsFiles.get(i).getPath())
+ LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath())
.setDeleteAfterLoad(isDeleteAfterLoad)
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
.orElse(null);
@@ -505,6 +507,8 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
if (isAutoCreateSchemaOrVerifySchemaEnabled) {
getOrCreateTreeSchemaVerifier().autoCreateAndVerify(reader, device2TimeseriesMetadata);
+ } else {
+ checkTreeWritePermission(device2TimeseriesMetadata);
}
// TODO: how to get the correct write point count when
@@ -522,6 +526,22 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
addWritePointCount(writePointCount);
}
+ private void checkTreeWritePermission(
+ final Map<IDeviceID, List<TimeseriesMetadata>> device2TimeseriesMetadataList)
+ throws AuthException {
+ for (final Map.Entry<IDeviceID, List<TimeseriesMetadata>> entry :
+ device2TimeseriesMetadataList.entrySet()) {
+ final IDeviceID device = entry.getKey();
+ for (final TimeseriesMetadata timeseriesMetadata : entry.getValue()) {
+ if (!TSDataType.VECTOR.equals(timeseriesMetadata.getTsDataType())
+ && !timeseriesMetadata.getMeasurementId().isEmpty()) {
+ TreeSchemaAutoCreatorAndVerifier.checkWriteDataPermission(
+ this, device, timeseriesMetadata);
+ }
+ }
+ }
+ }
+
private void doAnalyzeSingleTableFile(
final File tsFile,
final TsFileSequenceReader reader,
@@ -711,14 +731,15 @@ public class LoadTsFileAnalyzer implements AutoCloseable {
isTableModelTsFile.get(i)
? loadTsFileDataTypeConverter
.convertForTableModel(
- new LoadTsFile(null, tsFiles.get(i).getPath(), Collections.emptyMap())
+ LoadTsFile.createUnchecked(
+ null, tsFiles.get(i).getPath(), Collections.emptyMap())
.setDatabase(databaseForTableData)
.setDeleteAfterLoad(isDeleteAfterLoad)
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
.orElse(null)
: loadTsFileDataTypeConverter
.convertForTreeModel(
- new LoadTsFileStatement(tsFiles.get(i).getPath())
+ LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath())
.setDeleteAfterLoad(isDeleteAfterLoad)
.setConvertOnTypeMismatch(isConvertOnTypeMismatch))
.orElse(null);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java
index de56a8603c5..d8fbb9c80d7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/TreeSchemaAutoCreatorAndVerifier.java
@@ -149,21 +149,7 @@ public class TreeSchemaAutoCreatorAndVerifier {
// not a timeseries, skip
} else {
- // check WRITE_DATA permission of timeseries
- long startTime = System.nanoTime();
- try {
- UserEntity userEntity = loadTsFileAnalyzer.context.getSession().getUserEntity();
- TSStatus status =
- AuthorityChecker.getAccessControl()
- .checkFullPathWriteDataPermission(
- userEntity, device, timeseriesMetadata.getMeasurementId());
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- throw new AuthException(
- TSStatusCode.representOf(status.getCode()), status.getMessage());
- }
- } finally {
- PerformanceOverviewMetrics.getInstance().recordAuthCost(System.nanoTime() - startTime);
- }
+ checkWriteDataPermission(loadTsFileAnalyzer, device, timeseriesMetadata);
final Pair<CompressionType, TSEncoding> compressionEncodingPair =
reader.readTimeseriesCompressionTypeAndEncoding(timeseriesMetadata);
schemaCache.addTimeSeries(
@@ -187,6 +173,26 @@ public class TreeSchemaAutoCreatorAndVerifier {
}
}
+ static void checkWriteDataPermission(
+ final LoadTsFileAnalyzer loadTsFileAnalyzer,
+ final IDeviceID device,
+ final TimeseriesMetadata timeseriesMetadata)
+ throws AuthException {
+ final long startTime = System.nanoTime();
+ try {
+ final UserEntity userEntity = loadTsFileAnalyzer.context.getSession().getUserEntity();
+ final TSStatus status =
+ AuthorityChecker.getAccessControl()
+ .checkFullPathWriteDataPermission(
+ userEntity, device, timeseriesMetadata.getMeasurementId());
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new AuthException(TSStatusCode.representOf(status.getCode()), status.getMessage());
+ }
+ } finally {
+ PerformanceOverviewMetrics.getInstance().recordAuthCost(System.nanoTime() - startTime);
+ }
+ }
+
/**
* This can only be invoked after all timeseries in the current tsfile have been processed.
* Otherwise, the isAligned status may be wrong.
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
index 6d3326dd402..87803d70d79 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.queryengine.plan.relational.analyzer;
import org.apache.iotdb.calc.plan.relational.metadata.CommonMetadataUtils;
+import org.apache.iotdb.commons.auth.entity.PrivilegeType;
import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.commons.exception.SemanticException;
import org.apache.iotdb.commons.queryengine.common.SessionInfo;
@@ -830,6 +831,10 @@ public class StatementAnalyzer {
@Override
public Scope visitLoadTsFile(final LoadTsFile node, final Optional<Scope> scope) {
queryContext.setQueryType(QueryType.OTHER);
+ accessControl.checkMissingPrivileges(
+ sessionContext.getUserName(),
+ Collections.singletonList(PrivilegeType.LOAD_TSFILE),
+ queryContext);
try (final LoadTsFileAnalyzer loadTsFileAnalyzer =
new LoadTsFileAnalyzer(node, node.isGeneratedByPipe(), queryContext)) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TableModelPrivilege.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TableModelPrivilege.java
index d06a08ef026..3fc99e55f41 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TableModelPrivilege.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TableModelPrivilege.java
@@ -32,6 +32,7 @@ public enum TableModelPrivilege {
SYSTEM,
SECURITY,
AUDIT,
+ LOAD_TSFILE,
// scope privilege
CREATE,
@@ -65,6 +66,8 @@ public enum TableModelPrivilege {
return PrivilegeType.SECURITY;
case AUDIT:
return PrivilegeType.AUDIT;
+ case LOAD_TSFILE:
+ return PrivilegeType.LOAD_TSFILE;
default:
throw new IllegalStateException("Unexpected value:" + this);
}
@@ -94,6 +97,8 @@ public enum TableModelPrivilege {
return TableModelPrivilege.SECURITY;
case AUDIT:
return TableModelPrivilege.AUDIT;
+ case LOAD_TSFILE:
+ return TableModelPrivilege.LOAD_TSFILE;
default:
throw new IllegalStateException("Unexpected value:" + privilegeType);
}
@@ -109,6 +114,7 @@ public enum TableModelPrivilege {
return AuditLogOperation.QUERY;
case INSERT:
case DELETE:
+ case LOAD_TSFILE:
return AuditLogOperation.DML;
case MANAGE_ROLE:
case MANAGE_USER:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java
index 7b4655c79a8..e525a5e061d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java
@@ -169,6 +169,7 @@ import org.apache.iotdb.rpc.TSStatusCode;
import com.google.common.collect.ImmutableList;
+import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -1149,8 +1150,14 @@ public class TreeAccessCheckVisitor extends StatementVisitor<TSStatus, TreeAcces
@Override
public TSStatus visitLoadFile(LoadTsFileStatement statement, TreeAccessCheckContext context) {
- // no need to check here, it will be checked in process phase
- return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ return checkGlobalAuth(
+ context.setAuditLogOperation(AuditLogOperation.DML),
+ PrivilegeType.LOAD_TSFILE,
+ () ->
+ statement.getTsFiles().stream()
+ .map(File::getPath)
+ .collect(Collectors.toList())
+ .toString());
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java
index 03de9a9ccb1..dc1a549f636 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java
@@ -72,6 +72,19 @@ public class LoadTsFile extends Statement {
private boolean needDecode4TimeColumn;
public LoadTsFile(NodeLocation location, String filePath, Map<String, String> loadAttributes) {
+ this(location, filePath, loadAttributes, true);
+ }
+
+ public static LoadTsFile createUnchecked(
+ NodeLocation location, String filePath, Map<String, String> loadAttributes) {
+ return new LoadTsFile(location, filePath, loadAttributes, false);
+ }
+
+ private LoadTsFile(
+ NodeLocation location,
+ String filePath,
+ Map<String, String> loadAttributes,
+ boolean validateSourcePath) {
super(location);
this.filePath = requireNonNull(filePath, "filePath is null");
@@ -89,7 +102,7 @@ public class LoadTsFile extends Statement {
try {
this.tsFiles =
org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement.processTsFile(
- new File(filePath));
+ new File(filePath), validateSourcePath);
this.resources = new ArrayList<>();
this.writePointCountList = new ArrayList<>();
this.isTableModel = new ArrayList<>(Collections.nCopies(this.tsFiles.size(), true));
@@ -283,7 +296,7 @@ public class LoadTsFile extends Statement {
final Map<String, String> properties = this.loadAttributes;
final LoadTsFile subStatement =
- new LoadTsFile(getLocation().orElse(null), filePath, properties);
+ LoadTsFile.createUnchecked(getLocation().orElse(null), filePath, properties);
// Copy all configuration properties
subStatement.databaseLevel = this.databaseLevel;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
index bc8446fd3da..6671dad3548 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
@@ -586,14 +586,14 @@ public class LoadTsFileScheduler implements IScheduler {
failedNode.isTableModel()
? loadTsFileDataTypeConverter
.convertForTableModel(
- new LoadTsFile(null, filePath, Collections.emptyMap())
+ LoadTsFile.createUnchecked(null, filePath, Collections.emptyMap())
.setDatabase(failedNode.getDatabase())
.setDeleteAfterLoad(failedNode.isDeleteAfterLoad())
.setConvertOnTypeMismatch(true))
.orElse(null)
: loadTsFileDataTypeConverter
.convertForTreeModel(
- new LoadTsFileStatement(filePath)
+ LoadTsFileStatement.createUnchecked(filePath)
.setDeleteAfterLoad(failedNode.isDeleteAfterLoad())
.setConvertOnTypeMismatch(true))
.orElse(null);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
index 512a62fdae7..a846c7feb43 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
@@ -35,7 +35,9 @@ import org.apache.tsfile.common.constant.TsFileConstant;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.nio.file.Path;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -71,6 +73,15 @@ public class LoadTsFileStatement extends Statement {
private boolean needDecode4TimeColumn;
public LoadTsFileStatement(String filePath) throws FileNotFoundException {
+ this(filePath, true);
+ }
+
+ public static LoadTsFileStatement createUnchecked(String filePath) throws FileNotFoundException {
+ return new LoadTsFileStatement(filePath, false);
+ }
+
+ private LoadTsFileStatement(String filePath, boolean validateSourcePath)
+ throws FileNotFoundException {
this.file = new File(filePath).getAbsoluteFile();
this.databaseLevel = IoTDBDescriptor.getInstance().getConfig().getDefaultDatabaseLevel();
this.verifySchema = true;
@@ -80,7 +91,7 @@ public class LoadTsFileStatement extends Statement {
IoTDBDescriptor.getInstance().getConfig().getLoadTabletConversionThresholdBytes();
this.autoCreateDatabase = IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled();
- this.tsFiles = processTsFile(file);
+ this.tsFiles = processTsFile(file, validateSourcePath);
this.resources = new ArrayList<>();
this.writePointCountList = new ArrayList<>();
this.isTableModel = new ArrayList<>(Collections.nCopies(this.tsFiles.size(), false));
@@ -88,6 +99,15 @@ public class LoadTsFileStatement extends Statement {
}
public static List<File> processTsFile(final File file) throws FileNotFoundException {
+ return processTsFile(file, true);
+ }
+
+ public static List<File> processTsFile(final File file, final boolean validateSourcePath)
+ throws FileNotFoundException {
+ if (validateSourcePath) {
+ validateLoadSourcePath(file);
+ }
+
final List<File> tsFiles = new ArrayList<>();
if (file.isFile()) {
tsFiles.add(file);
@@ -98,7 +118,7 @@ public class LoadTsFileStatement extends Statement {
"Can not find %s on this machine, notice that load can only handle files on this machine.",
file.getPath()));
}
- tsFiles.addAll(findAllTsFile(file));
+ tsFiles.addAll(findAllTsFile(file, validateSourcePath));
}
sortTsFiles(tsFiles);
return tsFiles;
@@ -120,7 +140,8 @@ public class LoadTsFileStatement extends Statement {
this.statementType = StatementType.MULTI_BATCH_INSERT;
}
- private static List<File> findAllTsFile(File file) {
+ private static List<File> findAllTsFile(File file, boolean validateSourcePath)
+ throws FileNotFoundException {
final File[] files = file.listFiles();
if (files == null) {
return Collections.emptyList();
@@ -128,15 +149,50 @@ public class LoadTsFileStatement extends Statement {
final List<File> tsFiles = new ArrayList<>();
for (File nowFile : files) {
+ if (validateSourcePath) {
+ validateLoadSourcePath(nowFile);
+ }
if (nowFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) {
tsFiles.add(nowFile);
} else if (nowFile.isDirectory()) {
- tsFiles.addAll(findAllTsFile(nowFile));
+ tsFiles.addAll(findAllTsFile(nowFile, validateSourcePath));
}
}
return tsFiles;
}
+ public static void validateLoadSourcePath(final String filePath) throws FileNotFoundException {
+ validateLoadSourcePath(new File(filePath));
+ }
+
+ private static void validateLoadSourcePath(final File file) throws FileNotFoundException {
+ final Path sourcePath = canonicalPath(file);
+ final String[] allowedDirs =
+ IoTDBDescriptor.getInstance().getConfig().getLoadTsFileAllowedDirs();
+
+ for (final String allowedDir : allowedDirs) {
+ if (sourcePath.startsWith(canonicalPath(new File(allowedDir)))) {
+ return;
+ }
+ }
+
+ throw new FileNotFoundException(
+ String.format(
+ "Load TsFile source path %s is outside allowed directories %s.",
+ sourcePath, Arrays.toString(allowedDirs)));
+ }
+
+ private static Path canonicalPath(final File file) throws FileNotFoundException {
+ try {
+ return file.getCanonicalFile().toPath();
+ } catch (final IOException e) {
+ throw new FileNotFoundException(
+ String.format(
+ "Failed to resolve canonical path for Load TsFile source %s: %s",
+ file.getPath(), e.getMessage()));
+ }
+ }
+
private static void sortTsFiles(List<File> files) {
files.sort(
(o1, o2) -> {
@@ -389,7 +445,7 @@ public class LoadTsFileStatement extends Statement {
loadAttributes.put(PIPE_GENERATED_KEY, String.valueOf(true));
}
- return new LoadTsFile(null, file.getAbsolutePath(), loadAttributes);
+ return LoadTsFile.createUnchecked(null, file.getAbsolutePath(), loadAttributes);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java
index d0be2ead5cb..67fa0fff300 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java
@@ -219,7 +219,8 @@ public class ActiveLoadTsFileLoader {
final ActiveLoadPendingQueue.ActiveLoadEntry entry, final IClientSession session)
throws FileNotFoundException {
final File tsFile = new File(entry.getFile());
- final LoadTsFileStatement statement = new LoadTsFileStatement(tsFile.getAbsolutePath());
+ final LoadTsFileStatement statement =
+ LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath());
final List<File> files = statement.getTsFiles();
statement.setDeleteAfterLoad(true);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/TreeAccessTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/TreeAccessTest.java
index af8746fae3e..6eff04b0e47 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/TreeAccessTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/TreeAccessTest.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.auth.entity.User;
import org.apache.iotdb.db.queryengine.plan.relational.security.TreeAccessCheckContext;
import org.apache.iotdb.db.queryengine.plan.relational.security.TreeAccessCheckVisitor;
import org.apache.iotdb.db.queryengine.plan.statement.AuthorType;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.AuthorStatement;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -33,6 +34,9 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
public class TreeAccessTest {
@Before
@@ -81,4 +85,37 @@ public class TreeAccessTest {
.visitAuthor(authorStatement, new TreeAccessCheckContext(10000L, "user1", ""))
.getCode());
}
+
+ @Test
+ public void testLoadTsFileRequiresLoadTsFilePrivilege() throws Exception {
+ final User mockUser = Mockito.mock(User.class);
+ Mockito.when(mockUser.getName()).thenReturn("loadUser");
+ Mockito.when(mockUser.getUserId()).thenReturn(10002L);
+ Mockito.when(mockUser.checkSysPrivilege(PrivilegeType.LOAD_TSFILE)).thenReturn(false);
+ Mockito.when(mockUser.checkSysPrivilege(PrivilegeType.SYSTEM)).thenReturn(false);
+ AuthorityChecker.getAuthorityFetcher()
+ .getAuthorCache()
+ .putUserCache(mockUser.getName(), mockUser);
+
+ final Path tsFile = Files.createTempFile("load-auth", ".tsfile");
+ try {
+ final LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(tsFile.toString());
+ final TreeAccessCheckVisitor treeAccessCheckVisitor = new TreeAccessCheckVisitor();
+
+ Assert.assertEquals(
+ TSStatusCode.NO_PERMISSION.getStatusCode(),
+ treeAccessCheckVisitor
+ .visitLoadFile(statement, new TreeAccessCheckContext(10002L, "loadUser", ""))
+ .getCode());
+
+ Mockito.when(mockUser.checkSysPrivilege(PrivilegeType.LOAD_TSFILE)).thenReturn(true);
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ treeAccessCheckVisitor
+ .visitLoadFile(statement, new TreeAccessCheckContext(10002L, "loadUser", ""))
+ .getCode());
+ } finally {
+ Files.deleteIfExists(tsFile);
+ }
+ }
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AuthorStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AuthorStatementTest.java
index 509c529b36d..bb43f35f316 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AuthorStatementTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AuthorStatementTest.java
@@ -238,6 +238,8 @@ public class AuthorStatementTest {
checkGrantPrivileges(true, Collections.singletonList(PrivilegeType.SECURITY), "", false, false);
checkGrantPrivileges(
true, Arrays.asList(PrivilegeType.SYSTEM, PrivilegeType.SECURITY), "", true, false);
+ checkGrantPrivileges(
+ true, Collections.singletonList(PrivilegeType.LOAD_TSFILE), "", true, false);
checkGrantPrivileges(
true, Collections.singletonList(PrivilegeType.SECURITY), "ANY", true, true);
@@ -301,6 +303,8 @@ public class AuthorStatementTest {
true, Arrays.asList(PrivilegeType.SYSTEM, PrivilegeType.SECURITY), "", false, false);
checkRevokePrivileges(
true, Arrays.asList(PrivilegeType.SYSTEM, PrivilegeType.SECURITY), "", true, false);
+ checkRevokePrivileges(
+ true, Collections.singletonList(PrivilegeType.LOAD_TSFILE), "", false, false);
// Illegal privileges combination
checkRevokePrivileges(
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java
index 941794bb074..25644e97c71 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.junit.Assert;
import org.junit.Test;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -38,10 +39,12 @@ public class LoadTsFileStatementTest {
public void testSubStatementsKeepDatabase() throws Exception {
final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
final int originalBatchSize = config.getLoadTsFileSubStatementBatchSize();
+ final String[] originalAllowedDirs = config.getLoadTsFileAllowedDirs().clone();
final Path tempDir = Files.createTempDirectory("load-tsfile-sub-statements");
try {
config.setLoadTsFileSubStatementBatchSize(1);
+ config.setLoadTsFileAllowedDirs(new String[] {tempDir.toString()});
Files.createFile(tempDir.resolve("a.tsfile"));
Files.createFile(tempDir.resolve("b.tsfile"));
@@ -54,10 +57,42 @@ public class LoadTsFileStatementTest {
subStatement -> Assert.assertEquals("test_db", subStatement.getDatabase()));
} finally {
config.setLoadTsFileSubStatementBatchSize(originalBatchSize);
+ config.setLoadTsFileAllowedDirs(originalAllowedDirs);
deleteRecursively(tempDir);
}
}
+ @Test
+ public void testLoadSourcePathMustBeInAllowedDirs() throws Exception {
+ final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ final String[] originalAllowedDirs = config.getLoadTsFileAllowedDirs().clone();
+ final Path allowedDir = Files.createTempDirectory("load-tsfile-allowed");
+ final Path deniedDir = Files.createTempDirectory("load-tsfile-denied");
+
+ try {
+ config.setLoadTsFileAllowedDirs(new String[] {allowedDir.toString()});
+ final Path deniedTsFile = Files.createFile(deniedDir.resolve("denied.tsfile"));
+ final Path traversalTsFile =
+ allowedDir.resolve("..").resolve(deniedDir.getFileName()).resolve("denied.tsfile");
+
+ assertLoadSourcePathRejected(deniedTsFile);
+ assertLoadSourcePathRejected(traversalTsFile);
+ } finally {
+ config.setLoadTsFileAllowedDirs(originalAllowedDirs);
+ deleteRecursively(allowedDir);
+ deleteRecursively(deniedDir);
+ }
+ }
+
+ private static void assertLoadSourcePathRejected(final Path sourcePath) {
+ try {
+ new LoadTsFileStatement(sourcePath.toString());
+ Assert.fail("Expected disallowed LOAD TSFILE source path to be rejected.");
+ } catch (final FileNotFoundException e) {
+ Assert.assertTrue(e.getMessage().contains("outside allowed directories"));
+ }
+ }
+
private static void deleteRecursively(final Path path) throws IOException {
if (path == null || !Files.exists(path)) {
return;
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 69bcd275250..7606909a039 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -2182,6 +2182,12 @@ load_clean_up_task_execution_delay_time_seconds=1800
# Datatype: int
load_write_throughput_bytes_per_second=-1
+# Comma-separated list of directories from which user-issued LOAD TSFILE statements can read.
+# If empty, IoTDB only permits LOAD sources under the internal load TsFile directories.
+# effectiveMode: hot_reload
+# Datatype: String
+load_tsfile_allowed_dirs=
+
# Whether to enable the active listening mode for tsfile loading.
# effectiveMode: hot_reload
# Datatype: Boolean
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/PrivilegeType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/PrivilegeType.java
index 29dc5301a84..5d0b4359840 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/PrivilegeType.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/PrivilegeType.java
@@ -54,7 +54,8 @@ public enum PrivilegeType {
SYSTEM(PrivilegeModelType.SYSTEM),
SECURITY(PrivilegeModelType.SYSTEM),
- AUDIT(PrivilegeModelType.SYSTEM);
+ AUDIT(PrivilegeModelType.SYSTEM),
+ LOAD_TSFILE(PrivilegeModelType.SYSTEM);
private final PrivilegeModelType modelType;
@@ -134,6 +135,7 @@ public enum PrivilegeType {
switch (this) {
case MANAGE_USER:
case MANAGE_ROLE:
+ case LOAD_TSFILE:
case SYSTEM:
case SECURITY:
case AUDIT:
@@ -160,6 +162,7 @@ public enum PrivilegeType {
case USE_PIPE:
case MANAGE_DATABASE:
case EXTEND_TEMPLATE:
+ case LOAD_TSFILE:
return Arrays.asList(this, PrivilegeType.SYSTEM);
default:
return Collections.singletonList(this);
@@ -208,6 +211,7 @@ public enum PrivilegeType {
case WRITE_SCHEMA:
case INSERT:
case DELETE:
+ case LOAD_TSFILE:
return AuditLogOperation.DML;
case MANAGE_USER:
case MANAGE_ROLE:
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/AuthUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/AuthUtils.java
index 1aae1909ef2..f6c636251e7 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/AuthUtils.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/AuthUtils.java
@@ -568,6 +568,8 @@ public class AuthUtils {
return PrivilegeType.SECURITY;
case 12:
return PrivilegeType.AUDIT;
+ case 13:
+ return PrivilegeType.LOAD_TSFILE;
default:
// Not reach here.
LOGGER.warn("Not support position");
@@ -603,6 +605,8 @@ public class AuthUtils {
return 11;
case AUDIT:
return 12;
+ case LOAD_TSFILE:
+ return 13;
default:
return -1;
}
@@ -621,6 +625,7 @@ public class AuthUtils {
case USE_PIPE:
case MANAGE_DATABASE:
case EXTEND_TEMPLATE:
+ case LOAD_TSFILE:
return Arrays.asList(priv, PrivilegeType.SYSTEM);
default:
return Collections.singletonList(priv);
diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
index ca766464686..8e94f92ed69 100644
--- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
+++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
@@ -829,6 +829,7 @@ systemPrivilege
| MANAGE_ROLE
| SYSTEM
| SECURITY
+ | LOAD_TSFILE
;
objectPrivilege
@@ -1493,7 +1494,7 @@ nonReserved
| INDEX | INDEXES | IF | IGNORE | IMMEDIATE | INCLUDING | INITIAL | INPUT | INTERVAL | INVOKER | IO | ITERATE | ISOLATION
| JSON
| KEEP | KEY | KEYS | KILL
- | LANGUAGE | LAST | LATERAL | LEADING | LEAVE | LEVEL | LIMIT | LINEAR | LOAD | LOCAL | LOGICAL | LOOP
+ | LANGUAGE | LAST | LATERAL | LEADING | LEAVE | LEVEL | LIMIT | LINEAR | LOAD | LOAD_TSFILE | LOCAL | LOGICAL | LOOP
| MANAGE_ROLE | MANAGE_USER | MAP | MATCH | MATCHED | MATCHES | MATCH_RECOGNIZE | MATERIALIZED | MEASURES | MEMORY_THRESHOLD | METHOD | MERGE | MICROSECOND | MIGRATE | MILLISECOND | MINUTE | MODEL | MODELS | MODIFY | MONTH
| NANOSECOND | NESTED | NEXT | NFC | NFD | NFKC | NFKD | NO | NODEID | NONE | NULLIF | NULLS
| OBJECT | OF | OFFSET | OMIT | ONE | ONLY | OPTION | ORDINALITY | OUTPUT | OVER | OVERFLOW
@@ -1700,6 +1701,7 @@ LINEAR: 'LINEAR';
LIST: 'LIST';
LISTAGG: 'LISTAGG';
LOAD: 'LOAD';
+LOAD_TSFILE: 'LOAD_TSFILE';
LOADED: 'LOADED';
LOCAL: 'LOCAL';
LOCALTIME: 'LOCALTIME';