You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2024/12/10 03:02:03 UTC
(iotdb) 01/01: Supprt trusted_uri_pattern in config file
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/TrustedURI
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2855d8223a5383ce885c62b267025bf0684ec915
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Tue Dec 10 11:01:48 2024 +0800
Supprt trusted_uri_pattern in config file
---
.../iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java | 12 ++++++++
.../it/db/it/udf/scalar/IoTDBScalarFunctionIT.java | 13 +++++++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 23 +++++++++++++++
.../execution/config/TableConfigTaskVisitor.java | 20 +++++++++++--
.../execution/config/TreeConfigTaskVisitor.java | 33 ++++++++++++++++++++--
.../config/executor/ClusterConfigTaskExecutor.java | 18 ++++++------
.../conf/iotdb-system.properties.template | 5 ++++
.../apache/iotdb/commons/conf/CommonConfig.java | 11 ++++++++
.../commons/executable/ExecutableManager.java | 5 ++++
9 files changed, 126 insertions(+), 14 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
index 5de421ebd2a..71555bce0e3 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
@@ -152,4 +152,16 @@ public class IoTDBUDFBlockQueryIT {
fail(throwable.getMessage());
}
}
+
+ @Test
+ public void testUntrustedUri() {
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "CREATE FUNCTION two_sum AS 'org.apache.iotdb.db.query.udf.example.TwoSum' USING URI 'https://alioss.timecho.com/upload/library-udf.jar'");
+ fail("should fail");
+ } catch (SQLException throwable) {
+ assertTrue(throwable.getMessage().contains("701: Untrusted uri "));
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/scalar/IoTDBScalarFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/scalar/IoTDBScalarFunctionIT.java
index 7d7f1cbbe35..e1fcfafe2fa 100644
--- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/scalar/IoTDBScalarFunctionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/udf/scalar/IoTDBScalarFunctionIT.java
@@ -41,6 +41,7 @@ import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@RunWith(IoTDBTestRunner.class)
@@ -221,4 +222,16 @@ public class IoTDBScalarFunctionIT {
fail(e.getMessage());
}
}
+
+ @Test
+ public void testUntrustedUri() {
+ try (Connection connection = EnvFactory.getEnv().getTableConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(
+ "CREATE FUNCTION test as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull' USING URI 'https://alioss.timecho.com/upload/library-udf.jar'");
+ fail("should fail");
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("701: Untrusted uri "));
+ }
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index b652acf7329..48815bbeba1 100755
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -87,6 +87,7 @@ import java.util.Optional;
import java.util.Properties;
import java.util.ServiceLoader;
import java.util.Set;
+import java.util.regex.Pattern;
public class IoTDBDescriptor {
@@ -2870,6 +2871,28 @@ public class IoTDBDescriptor {
} else {
BinaryAllocator.getInstance().close(true);
}
+
+ // update trusted_uri_pattern
+ String trustedUriPattern =
+ Optional.ofNullable(
+ properties.getProperty(
+ "trusted_uri_pattern",
+ ConfigurationFileUtils.getConfigurationDefaultValue("trusted_uri_pattern")))
+ .map(String::trim)
+ .orElse(ConfigurationFileUtils.getConfigurationDefaultValue("trusted_uri_pattern"));
+ Pattern pattern;
+ if (trustedUriPattern != null) {
+ try {
+ pattern = Pattern.compile(trustedUriPattern);
+ } catch (Exception e) {
+ LOGGER.warn("Failed to parse trusted_uri_pattern {}", trustedUriPattern);
+ pattern = commonDescriptor.getConfig().getTrustedUriPattern();
+ }
+ } else {
+ pattern = commonDescriptor.getConfig().getTrustedUriPattern();
+ }
+ commonDescriptor.getConfig().setTrustedUriPattern(pattern);
+
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
index 5c6178830c3..c2ee7588162 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.queryengine.plan.execution.config;
import org.apache.iotdb.common.rpc.thrift.Model;
import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.executable.ExecutableManager;
import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.commons.schema.table.TsTable;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
@@ -166,6 +167,7 @@ import java.util.Optional;
import static org.apache.iotdb.commons.conf.IoTDBConstant.MAX_DATABASE_NAME_LENGTH;
import static org.apache.iotdb.commons.conf.IoTDBConstant.TTL_INFINITE;
+import static org.apache.iotdb.commons.executable.ExecutableManager.isUriTrusted;
import static org.apache.iotdb.commons.schema.table.TsTable.TABLE_ALLOWED_PROPERTIES;
import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY;
import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask.DATA_REGION_GROUP_NUM_KEY;
@@ -751,7 +753,14 @@ public class TableConfigTaskVisitor extends AstVisitor<IConfigTask, MPPQueryCont
@Override
protected IConfigTask visitCreatePipePlugin(CreatePipePlugin node, MPPQueryContext context) {
context.setQueryType(QueryType.WRITE);
- return new CreatePipePluginTask(node);
+ if (node.getUriString() != null && isUriTrusted(node.getUriString())) {
+ // 1. user specified uri and that uri is trusted
+ // 2. user doesn't specify uri
+ return new CreatePipePluginTask(node);
+ } else {
+ // user specified uri and that uri is not trusted
+ throw new SemanticException("Untrusted uri " + node.getUriString());
+ }
}
@Override
@@ -851,7 +860,14 @@ public class TableConfigTaskVisitor extends AstVisitor<IConfigTask, MPPQueryCont
@Override
protected IConfigTask visitCreateFunction(CreateFunction node, MPPQueryContext context) {
context.setQueryType(QueryType.WRITE);
- return new CreateFunctionTask(node);
+ if (node.getUriString().map(ExecutableManager::isUriTrusted).orElse(true)) {
+ // 1. user specified uri and that uri is trusted
+ // 2. user doesn't specify uri
+ return new CreateFunctionTask(node);
+ } else {
+ // user specified uri and that uri is not trusted
+ throw new SemanticException("Untrusted uri " + node.getUriString().get());
+ }
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java
index 5b1e513089c..b7fbad6ae28 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.queryengine.plan.execution.config;
import org.apache.iotdb.common.rpc.thrift.Model;
+import org.apache.iotdb.commons.executable.ExecutableManager;
import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
@@ -174,6 +175,8 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowThrottleQuot
import org.apache.tsfile.exception.NotImplementedException;
+import static org.apache.iotdb.commons.executable.ExecutableManager.isUriTrusted;
+
public class TreeConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQueryContext> {
@Override
@@ -319,7 +322,14 @@ public class TreeConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQuer
@Override
public IConfigTask visitCreateFunction(
CreateFunctionStatement createFunctionStatement, MPPQueryContext context) {
- return new CreateFunctionTask(createFunctionStatement);
+ if (createFunctionStatement.getUriString().map(ExecutableManager::isUriTrusted).orElse(true)) {
+ // 1. user specified uri and that uri is trusted
+ // 2. user doesn't specify uri
+ return new CreateFunctionTask(createFunctionStatement);
+ } else {
+ // user specified uri and that uri is not trusted
+ throw new SemanticException("Untrusted uri " + createFunctionStatement.getUriString().get());
+ }
}
@Override
@@ -337,7 +347,16 @@ public class TreeConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQuer
@Override
public IConfigTask visitCreateTrigger(
CreateTriggerStatement createTriggerStatement, MPPQueryContext context) {
- return new CreateTriggerTask(createTriggerStatement);
+ if (createTriggerStatement.isUsingURI()
+ && createTriggerStatement.getUriString() != null
+ && isUriTrusted(createTriggerStatement.getUriString())) {
+ // 1. user specified uri and that uri is trusted
+ // 2. user doesn't specify uri
+ return new CreateTriggerTask(createTriggerStatement);
+ } else {
+ // user specified uri and that uri is not trusted
+ throw new SemanticException("Untrusted uri " + createTriggerStatement.getUriString());
+ }
}
@Override
@@ -355,7 +374,15 @@ public class TreeConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQuer
@Override
public IConfigTask visitCreatePipePlugin(
CreatePipePluginStatement createPipePluginStatement, MPPQueryContext context) {
- return new CreatePipePluginTask(createPipePluginStatement);
+ if (createPipePluginStatement.getUriString() != null
+ && isUriTrusted(createPipePluginStatement.getUriString())) {
+ // 1. user specified uri and that uri is trusted
+ // 2. user doesn't specify uri
+ return new CreatePipePluginTask(createPipePluginStatement);
+ } else {
+ // user specified uri and that uri is not trusted
+ throw new SemanticException("Untrusted uri " + createPipePluginStatement.getUriString());
+ }
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
index 286bfe8b578..4e73064e58d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
@@ -541,7 +541,6 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
String jarFilePathUnderTempDir =
UDFExecutableManager.getInstance()
.getDirStringUnderTempRootByRequestId(resource.getRequestId())
- + File.separator
+ jarFileName;
// libRoot should be the path of the specified jar
libRoot = jarFilePathUnderTempDir;
@@ -568,12 +567,15 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
tCreateFunctionReq.setJarFile(jarFile);
tCreateFunctionReq.setJarMD5(jarMd5);
tCreateFunctionReq.setIsUsingURI(true);
- tCreateFunctionReq.setJarName(
- String.format(
- "%s-%s.%s",
- jarFileName.substring(0, jarFileName.lastIndexOf(".")),
- jarMd5,
- jarFileName.substring(jarFileName.lastIndexOf(".") + 1)));
+ int index = jarFileName.lastIndexOf(".");
+ if (index < 0) {
+ tCreateFunctionReq.setJarName(String.format("%s-%s", jarFileName, jarMd5));
+ } else {
+ tCreateFunctionReq.setJarName(
+ String.format(
+ "%s-%s.%s",
+ jarFileName.substring(0, index), jarMd5, jarFileName.substring(index + 1)));
+ }
}
FunctionType functionType = FunctionType.NONE;
@@ -728,7 +730,6 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
String jarFilePathUnderTempDir =
TriggerExecutableManager.getInstance()
.getDirStringUnderTempRootByRequestId(resource.getRequestId())
- + File.separator
+ jarFileName;
// libRoot should be the path of the specified jar
libRoot = jarFilePathUnderTempDir;
@@ -894,7 +895,6 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
final String jarFilePathUnderTempDir =
PipePluginExecutableManager.getInstance()
.getDirStringUnderTempRootByRequestId(resource.getRequestId())
- + File.separator
+ jarFileName;
// libRoot should be the path of the specified jar
libRoot = jarFilePathUnderTempDir;
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index c7b9115eea0..310e4ae5577 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1616,6 +1616,11 @@ author_cache_size=1000
# Datatype: int
author_cache_expire_time=30
+# A regex pattern representing trusted uri by system
+# effectiveMode: hot_reload
+# Datatype: regex
+trusted_uri_pattern=file:.*
+
####################
### UDF Configuration
####################
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index 2d1585d6525..8d30d0c6d80 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -36,6 +36,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
import static org.apache.iotdb.commons.conf.IoTDBConstant.MB;
@@ -343,6 +344,8 @@ public class CommonConfig {
private volatile long remoteWriteMaxRetryDurationInMs = 60000;
+ private volatile Pattern trustedUriPattern = Pattern.compile("file:.*");
+
CommonConfig() {
// Empty constructor
}
@@ -1529,4 +1532,12 @@ public class CommonConfig {
public void setLog2SizeClassGroup(int log2SizeClassGroup) {
this.log2SizeClassGroup = log2SizeClassGroup;
}
+
+ public Pattern getTrustedUriPattern() {
+ return trustedUriPattern;
+ }
+
+ public void setTrustedUriPattern(Pattern trustedUriPattern) {
+ this.trustedUriPattern = trustedUriPattern;
+ }
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java
index 435ee489234..b00276db38a 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ExecutableManager.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.commons.executable;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.trigger.exception.TriggerJarTooLargeException;
import org.apache.commons.io.FileUtils;
@@ -282,4 +283,8 @@ public class ExecutableManager {
public String getInstallDir() {
return libRoot + File.separator + INSTALL_DIR;
}
+
+ public static boolean isUriTrusted(String uri) {
+ return CommonDescriptor.getInstance().getConfig().getTrustedUriPattern().matcher(uri).matches();
+ }
}