You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2022/05/23 17:38:31 UTC
[iotdb] 17/18: basic structure of UDFManager
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8fe3c0738326b570d889c9e0bb00342c277eb9f5
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Mon May 23 19:00:11 2022 +0800
basic structure of UDFManager
---
.../resources/conf/iotdb-confignode.properties | 28 ++++++++--
.../iotdb/confignode/conf/ConfigNodeConf.java | 36 ++++++++++---
.../confignode/conf/ConfigNodeDescriptor.java | 5 +-
.../iotdb/confignode/manager/UDFManager.java | 23 +++++++-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 +
.../commons/udf/service/UDFExecutableFetcher.java | 62 ++++++++++++++++++++++
6 files changed, 142 insertions(+), 13 deletions(-)
diff --git a/confignode/src/assembly/resources/conf/iotdb-confignode.properties b/confignode/src/assembly/resources/conf/iotdb-confignode.properties
index b16a399ec3..8fb85749b6 100644
--- a/confignode/src/assembly/resources/conf/iotdb-confignode.properties
+++ b/confignode/src/assembly/resources/conf/iotdb-confignode.properties
@@ -212,9 +212,9 @@ target_confignode=0.0.0.0:22277
# proc_wal_dir=data/proc
-# UDF dir
+# UDF lib dir
# If this property is unset, system will save the data in the default relative path directory under
-# the UDF folder(i.e., %CONFIGNODE_HOME%/data/udf).
+# the UDF folder(i.e., %CONFIGNODE_HOME%/ext/udf).
#
# If it is absolute, system will save the data in exact location it points to.
# If it is relative, system will save the data in the relative path directory it indicates under the
@@ -225,11 +225,31 @@ target_confignode=0.0.0.0:22277
# For Window platform
# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is
# absolute. Otherwise, it is relative.
-# udf_dir=data\\udf
+# udf_lib_dir=ext\\udf
#
# For Linux platform
# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
-# udf_dir=data/udf
+# udf_lib_dir=ext/udf
+
+
+# temporary lib dir
+# If this property is unset, system will save the data in the default relative path directory under
+# the UDF folder(i.e., %CONFIGNODE_HOME%/ext/temporary).
+#
+# If it is absolute, system will save the data in exact location it points to.
+# If it is relative, system will save the data in the relative path directory it indicates under the
+# UDF folder.
+# Note: If data_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative
+# path.
+#
+# For Window platform
+# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is
+# absolute. Otherwise, it is relative.
+# temporary_lib_dir=ext\\temporary
+#
+# For Linux platform
+# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
+# temporary_lib_dir=ext/temporary
####################
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConf.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConf.java
index 5362f9f95b..0ca14d6dd5 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConf.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConf.java
@@ -103,9 +103,16 @@ public class ConfigNodeConf {
private String consensusDir =
ConfigNodeConstant.DATA_DIR + File.separator + ConfigNodeConstant.CONSENSUS_FOLDER;
- /** UDF directory, storage udf metadata and jars */
- private String udfDir =
- ConfigNodeConstant.DATA_DIR + File.separator + ConfigNodeConstant.UDF_FOLDER;
+ /** External lib directory, stores user-uploaded JAR files */
+ private String extLibDir = IoTDBConstant.EXT_FOLDER_NAME;
+
+ /** External lib directory for UDF, stores user-uploaded JAR files */
+ private String udfLibDir =
+ IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.UDF_FOLDER_NAME;
+
+ /** External temporary lib directory for storing downloaded JAR files */
+ private String temporaryLibDir =
+ IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.TMP_FOLDER_NAME;
/** Time partition interval in seconds */
private long timePartitionInterval = 604800;
@@ -149,6 +156,9 @@ public class ConfigNodeConf {
private void formulateFolders() {
systemDir = addHomeDir(systemDir);
consensusDir = addHomeDir(consensusDir);
+ extLibDir = addHomeDir(extLibDir);
+ udfLibDir = addHomeDir(udfLibDir);
+ temporaryLibDir = addHomeDir(temporaryLibDir);
}
private String addHomeDir(String dir) {
@@ -336,12 +346,24 @@ public class ConfigNodeConf {
this.systemDir = systemDir;
}
- public String getUdfDir() {
- return udfDir;
+ public String getSystemUdfDir() {
+ return getSystemDir() + File.separator + "udf" + File.separator;
+ }
+
+ public String getUdfLibDir() {
+ return udfLibDir;
+ }
+
+ public void setUdfLibDir(String udfLibDir) {
+ this.udfLibDir = udfLibDir;
+ }
+
+ public String getTemporaryLibDir() {
+ return temporaryLibDir;
}
- public void setUdfDir(String udfDir) {
- this.udfDir = udfDir;
+ public void setTemporaryLibDir(String temporaryLibDir) {
+ this.temporaryLibDir = temporaryLibDir;
}
public int getSchemaReplicationFactor() {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
index b5c58efd92..2a024dd3b0 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
@@ -175,7 +175,10 @@ public class ConfigNodeDescriptor {
conf.setConsensusDir(properties.getProperty("consensus_dir", conf.getConsensusDir()));
- conf.setUdfDir(properties.getProperty("udf_dir", conf.getUdfDir()));
+ conf.setUdfLibDir(properties.getProperty("udf_lib_dir", conf.getUdfLibDir()));
+
+ conf.setTemporaryLibDir(
+ properties.getProperty("temporary_lib_dir", conf.getTemporaryLibDir()));
conf.setTimePartitionInterval(
Long.parseLong(
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java
index b5fbc250aa..e26fedcccd 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java
@@ -20,18 +20,39 @@
package org.apache.iotdb.confignode.manager;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.udf.service.UDFClassLoaderManager;
+import org.apache.iotdb.commons.udf.service.UDFExecutableFetcher;
+import org.apache.iotdb.commons.udf.service.UDFRegistrationService;
+import org.apache.iotdb.confignode.conf.ConfigNodeConf;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
import org.apache.iotdb.confignode.persistence.UDFInfo;
+import java.io.IOException;
import java.util.List;
public class UDFManager {
+ private static final ConfigNodeConf CONFIG_NODE_CONF =
+ ConfigNodeDescriptor.getInstance().getConf();
+
+ private final UDFExecutableFetcher udfExecutableFetcher;
+ private final UDFClassLoaderManager udfClassLoaderManager;
+ private final UDFRegistrationService udfRegistrationService;
+
private final ConfigManager configManager;
private final UDFInfo udfInfo;
- public UDFManager(ConfigManager configManager, UDFInfo udfInfo) {
+ public UDFManager(ConfigManager configManager, UDFInfo udfInfo) throws IOException {
this.configManager = configManager;
this.udfInfo = udfInfo;
+
+ udfExecutableFetcher =
+ UDFExecutableFetcher.setupAndGetInstance(
+ CONFIG_NODE_CONF.getTemporaryLibDir(), CONFIG_NODE_CONF.getUdfLibDir());
+ udfClassLoaderManager =
+ UDFClassLoaderManager.setupAndGetInstance(CONFIG_NODE_CONF.getUdfLibDir());
+ udfRegistrationService =
+ UDFRegistrationService.setupAndGetInstance(CONFIG_NODE_CONF.getSystemUdfDir());
}
public TSStatus createFunction(String udfName, String className, List<String> uris) {
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index 1e033baf64..566ba957c8 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@ -172,6 +172,7 @@ public class IoTDBConstant {
public static final String TRACING_LOG = "tracing.txt";
public static final String EXT_FOLDER_NAME = "ext";
public static final String UDF_FOLDER_NAME = "udf";
+ public static final String TMP_FOLDER_NAME = "temporary";
public static final String TRIGGER_FOLDER_NAME = "trigger";
public static final String MQTT_FOLDER_NAME = "mqtt";
public static final String WAL_FOLDER_NAME = "wal";
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableFetcher.java b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableFetcher.java
new file mode 100644
index 0000000000..d04b1819a2
--- /dev/null
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableFetcher.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.udf.service;
+
+import org.apache.iotdb.commons.file.SystemFileFactory;
+
+import org.apache.commons.io.FileUtils;
+
+import java.io.File;
+import java.io.IOException;
+
+public class UDFExecutableFetcher {
+
+ private final String temporaryLibRoot;
+ private final String extLibRoot;
+
+ private UDFExecutableFetcher(String temporaryLibRoot, String extLibRoot) {
+ this.temporaryLibRoot = temporaryLibRoot;
+ this.extLibRoot = extLibRoot;
+ }
+
+ private static UDFExecutableFetcher INSTANCE = null;
+
+ public static synchronized UDFExecutableFetcher setupAndGetInstance(
+ String temporaryLibRoot, String extLibRoot) throws IOException {
+ if (INSTANCE == null) {
+ makeDirIfNecessary(temporaryLibRoot);
+ makeDirIfNecessary(extLibRoot);
+ INSTANCE = new UDFExecutableFetcher(temporaryLibRoot, extLibRoot);
+ }
+ return INSTANCE;
+ }
+
+ private static void makeDirIfNecessary(String dir) throws IOException {
+ File file = SystemFileFactory.INSTANCE.getFile(dir);
+ if (file.exists() && file.isDirectory()) {
+ return;
+ }
+ FileUtils.forceMkdir(file);
+ }
+
+ public static UDFExecutableFetcher getInstance() {
+ return INSTANCE;
+ }
+}