You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2024/12/12 07:04:27 UTC
(iotdb) branch TrustedUri-1.3 created (now cc335c0d1bb)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch TrustedUri-1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at cc335c0d1bb Supprt trusted_uri_pattern in config file
This branch includes the following new commits:
new cc335c0d1bb Supprt trusted_uri_pattern in config file
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: Supprt trusted_uri_pattern in config file
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch TrustedUri-1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit cc335c0d1bb8b192f8c0760609ea4e7f17b16a2b
Author: Jackie Tien <ja...@gmail.com>
AuthorDate: Thu Dec 12 14:33:40 2024 +0800
Supprt trusted_uri_pattern in config file
(cherry picked from commit 37450ec160b3f66e46c2015eb758b549877753b1)
---
.../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 1 +
.../iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java | 12 ++++++
.../pipe/it/autocreate/IoTDBPipeLifeCycleIT.java | 2 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 23 +++++++++++
.../plan/execution/config/ConfigTaskVisitor.java | 44 ++++++++++++++++++++--
.../config/executor/ClusterConfigTaskExecutor.java | 18 ++++-----
.../db/queryengine/plan/parser/ASTVisitor.java | 2 +-
.../conf/iotdb-system.properties.template | 6 +++
.../apache/iotdb/commons/conf/CommonConfig.java | 11 ++++++
.../commons/executable/ExecutableManager.java | 5 +++
11 files changed, 110 insertions(+), 16 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java
index 0306b1a5225..265fbf4e301 100644
--- a/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/ainode/it/AINodeBasicIT.java
@@ -56,6 +56,7 @@ public class AINodeBasicIT {
static String[] sqls =
new String[] {
+ "set configuration \"trusted_uri_pattern\"='.*'",
"create model identity using uri \"" + MODEL_PATH + "\"",
"CREATE DATABASE root.AI.data",
"CREATE TIMESERIES root.AI.data.s0 WITH DATATYPE=FLOAT, ENCODING=RLE",
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
index 5de421ebd2a..71555bce0e3 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
@@ -152,4 +152,16 @@ public class IoTDBUDFBlockQueryIT {
fail(throwable.getMessage());
}
}
+
+ @Test
+ public void testUntrustedUri() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "CREATE FUNCTION two_sum AS 'org.apache.iotdb.db.query.udf.example.TwoSum' USING URI 'https://alioss.timecho.com/upload/library-udf.jar'");
+ fail("should fail");
+ } catch (SQLException throwable) {
+ assertTrue(throwable.getMessage().contains("701: Untrusted uri "));
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java
index 02abcde0a12..7ad604e01d8 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java
@@ -846,7 +846,7 @@ public class IoTDBPipeLifeCycleIT extends AbstractPipeDualAutoIT {
assertNonQueryTestFail(
senderEnv,
"create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'",
- "1603: The scheme of URI is not set, please specify the scheme of URI.",
+ "701: Untrusted uri xxx",
"test",
"test123");
tryExecuteNonQueryWithRetry(senderEnv, "drop pipePlugin TestProcessor", "test", "test123");
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
index 25bff7e5499..d0bffc17ad3 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
@@ -672,7 +672,7 @@ showSubscriptions
// AI Model =========================================================================================
// ---- Create Model
createModel
- : CREATE MODEL modelName=identifier USING URI modelUri=STRING_LITERAL
+ : CREATE MODEL modelName=identifier uriClause
;
windowFunction
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index b5e29e5c901..0052cc82140 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -86,6 +86,7 @@ import java.util.Optional;
import java.util.Properties;
import java.util.ServiceLoader;
import java.util.Set;
+import java.util.regex.Pattern;
public class IoTDBDescriptor {
@@ -2003,6 +2004,28 @@ public class IoTDBDescriptor {
} else {
BinaryAllocator.getInstance().close(true);
}
+
+ // update trusted_uri_pattern
+ String trustedUriPattern =
+ Optional.ofNullable(
+ properties.getProperty(
+ "trusted_uri_pattern",
+ ConfigurationFileUtils.getConfigurationDefaultValue("trusted_uri_pattern")))
+ .map(String::trim)
+ .orElse(ConfigurationFileUtils.getConfigurationDefaultValue("trusted_uri_pattern"));
+ Pattern pattern;
+ if (trustedUriPattern != null) {
+ try {
+ pattern = Pattern.compile(trustedUriPattern);
+ } catch (Exception e) {
+ LOGGER.warn("Failed to parse trusted_uri_pattern {}", trustedUriPattern);
+ pattern = commonDescriptor.getConfig().getTrustedUriPattern();
+ }
+ } else {
+ pattern = commonDescriptor.getConfig().getTrustedUriPattern();
+ }
+ commonDescriptor.getConfig().setTrustedUriPattern(pattern);
+
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java
index b928d472234..2f3b758e77a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.queryengine.plan.execution.config;
+import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.CountDatabaseTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.CountTimeSlotListTask;
@@ -171,6 +172,8 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowThrottleQuot
import org.apache.tsfile.exception.NotImplementedException;
+import static org.apache.iotdb.commons.executable.ExecutableManager.isUriTrusted;
+
public class ConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQueryContext> {
@Override
@@ -316,7 +319,16 @@ public class ConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQueryCon
@Override
public IConfigTask visitCreateFunction(
CreateFunctionStatement createFunctionStatement, MPPQueryContext context) {
- return new CreateFunctionTask(createFunctionStatement);
+ if (!createFunctionStatement.isUsingURI()
+ || (createFunctionStatement.getUriString() != null
+ && isUriTrusted(createFunctionStatement.getUriString()))) {
+ // 1. user specified uri and that uri is trusted
+ // 2. user doesn't specify uri
+ return new CreateFunctionTask(createFunctionStatement);
+ } else {
+ // user specified uri and that uri is not trusted
+ throw new SemanticException("Untrusted uri " + createFunctionStatement.getUriString());
+ }
}
@Override
@@ -334,7 +346,16 @@ public class ConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQueryCon
@Override
public IConfigTask visitCreateTrigger(
CreateTriggerStatement createTriggerStatement, MPPQueryContext context) {
- return new CreateTriggerTask(createTriggerStatement);
+ if (!createTriggerStatement.isUsingURI()
+ || (createTriggerStatement.getUriString() != null
+ && isUriTrusted(createTriggerStatement.getUriString()))) {
+ // 1. user specified uri and that uri is trusted
+ // 2. user doesn't specify uri
+ return new CreateTriggerTask(createTriggerStatement);
+ } else {
+ // user specified uri and that uri is not trusted
+ throw new SemanticException("Untrusted uri " + createTriggerStatement.getUriString());
+ }
}
@Override
@@ -352,7 +373,15 @@ public class ConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQueryCon
@Override
public IConfigTask visitCreatePipePlugin(
CreatePipePluginStatement createPipePluginStatement, MPPQueryContext context) {
- return new CreatePipePluginTask(createPipePluginStatement);
+ if (createPipePluginStatement.getUriString() != null
+ && isUriTrusted(createPipePluginStatement.getUriString())) {
+ // 1. user specified uri and that uri is trusted
+ // 2. user doesn't specify uri
+ return new CreatePipePluginTask(createPipePluginStatement);
+ } else {
+ // user specified uri and that uri is not trusted
+ throw new SemanticException("Untrusted uri " + createPipePluginStatement.getUriString());
+ }
}
@Override
@@ -598,7 +627,14 @@ public class ConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQueryCon
@Override
public IConfigTask visitCreateModel(
CreateModelStatement createModelStatement, MPPQueryContext context) {
- return new CreateModelTask(createModelStatement, context);
+ if (createModelStatement.getUri() != null && isUriTrusted(createModelStatement.getUri())) {
+ // 1. user specified uri and that uri is trusted
+ // 2. user doesn't specify uri
+ return new CreateModelTask(createModelStatement, context);
+ } else {
+ // user specified uri and that uri is not trusted
+ throw new SemanticException("Untrusted uri " + createModelStatement.getUri());
+ }
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
index 618a9cd53dc..f9e824a87b6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
@@ -473,7 +473,6 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
String jarFilePathUnderTempDir =
UDFExecutableManager.getInstance()
.getDirStringUnderTempRootByRequestId(resource.getRequestId())
- + File.separator
+ jarFileName;
// libRoot should be the path of the specified jar
libRoot = jarFilePathUnderTempDir;
@@ -506,12 +505,15 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
tCreateFunctionReq.setJarFile(jarFile);
tCreateFunctionReq.setJarMD5(jarMd5);
tCreateFunctionReq.setIsUsingURI(true);
- tCreateFunctionReq.setJarName(
- String.format(
- "%s-%s.%s",
- jarFileName.substring(0, jarFileName.lastIndexOf(".")),
- jarMd5,
- jarFileName.substring(jarFileName.lastIndexOf(".") + 1)));
+ int index = jarFileName.lastIndexOf(".");
+ if (index < 0) {
+ tCreateFunctionReq.setJarName(String.format("%s-%s", jarFileName, jarMd5));
+ } else {
+ tCreateFunctionReq.setJarName(
+ String.format(
+ "%s-%s.%s",
+ jarFileName.substring(0, index), jarMd5, jarFileName.substring(index + 1)));
+ }
}
// try to create instance, this request will fail if creation is not successful
@@ -645,7 +647,6 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
String jarFilePathUnderTempDir =
TriggerExecutableManager.getInstance()
.getDirStringUnderTempRootByRequestId(resource.getRequestId())
- + File.separator
+ jarFileName;
// libRoot should be the path of the specified jar
libRoot = jarFilePathUnderTempDir;
@@ -811,7 +812,6 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
final String jarFilePathUnderTempDir =
PipePluginExecutableManager.getInstance()
.getDirStringUnderTempRootByRequestId(resource.getRequestId())
- + File.separator
+ jarFileName;
// libRoot should be the path of the specified jar
libRoot = jarFilePathUnderTempDir;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
index 8796c4e8b65..3c2e18fad17 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
@@ -1327,7 +1327,7 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
String modelName = ctx.modelName.getText();
validateModelName(modelName);
createModelStatement.setModelName(parseIdentifier(modelName));
- createModelStatement.setUri(ctx.modelUri.getText());
+ createModelStatement.setUri(ctx.uriClause().uri().getText());
return createModelStatement;
}
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 4cde7f8eff2..6253e50c85c 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1587,6 +1587,12 @@ author_cache_size=1000
# Datatype: int
author_cache_expire_time=30
+# A regex pattern representing trusted uri by system
+# effectiveMode: hot_reload
+# Datatype: regex
+# If you want to allow all URIs, you can specify it as .*
+trusted_uri_pattern=file:.*
+
####################
### UDF Configuration
####################
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index 2d1585d6525..8d30d0c6d80 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -36,6 +36,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
import static org.apache.iotdb.commons.conf.IoTDBConstant.MB;
@@ -343,6 +344,8 @@ public class CommonConfig {
private volatile long remoteWriteMaxRetryDurationInMs = 60000;
+ private volatile Pattern trustedUriPattern = Pattern.compile("file:.*");
+
CommonConfig() {
// Empty constructor
}
@@ -1529,4 +1532,12 @@ public class CommonConfig {
public void setLog2SizeClassGroup(int log2SizeClassGroup) {
this.log2SizeClassGroup = log2SizeClassGroup;
}
+
+ public Pattern getTrustedUriPattern() {
+ return trustedUriPattern;
+ }
+
+ public void setTrustedUriPattern(Pattern trustedUriPattern) {
+ this.trustedUriPattern = trustedUriPattern;
+ }
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java
index 435ee489234..b00276db38a 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.commons.executable;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.trigger.exception.TriggerJarTooLargeException;
import org.apache.commons.io.FileUtils;
@@ -282,4 +283,8 @@ public class ExecutableManager {
public String getInstallDir() {
return libRoot + File.separator + INSTALL_DIR;
}
+
+ public static boolean isUriTrusted(String uri) {
+ return CommonDescriptor.getInstance().getConfig().getTrustedUriPattern().matcher(uri).matches();
+ }
}