You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2022/05/23 17:38:14 UTC
[iotdb] branch iotdb-3227 created (now c9355e5d54)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a change to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at c9355e5d54 register function in UDFInfo
This branch includes the following new commits:
new b53039c021 g4 & statement
new fc57e0e8b3 Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
new 97c4924480 ASTVisitor
new b2fe9293ab visit task
new dfb73e801a /confignode.thrift
new ae8a2b9882 datanode -> confignode
new bb8d17a0ad ConfigNodeRPCServiceProcessor -> ConfigManager
new ab0fa0f6da Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
new d7a195f819 ConfigManager ConfigRequestExecutor
new fe5652865f merge master and fix conflicts
new d451ab0ef5 fix conflicts
new 9d5ce589ec fix compile error
new 00e6a63378 ConfigManager
new 966a6d30d0 Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
new 6fe0aff57b Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
new 9a56cd3fbb Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
new 8fe3c07383 basic structure of UDFManager
new c9355e5d54 register function in UDFInfo
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 17/18: basic structure of UDFManager
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8fe3c0738326b570d889c9e0bb00342c277eb9f5
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Mon May 23 19:00:11 2022 +0800
basic structure of UDFManager
---
.../resources/conf/iotdb-confignode.properties | 28 ++++++++--
.../iotdb/confignode/conf/ConfigNodeConf.java | 36 ++++++++++---
.../confignode/conf/ConfigNodeDescriptor.java | 5 +-
.../iotdb/confignode/manager/UDFManager.java | 23 +++++++-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 +
.../commons/udf/service/UDFExecutableFetcher.java | 62 ++++++++++++++++++++++
6 files changed, 142 insertions(+), 13 deletions(-)
diff --git a/confignode/src/assembly/resources/conf/iotdb-confignode.properties b/confignode/src/assembly/resources/conf/iotdb-confignode.properties
index b16a399ec3..8fb85749b6 100644
--- a/confignode/src/assembly/resources/conf/iotdb-confignode.properties
+++ b/confignode/src/assembly/resources/conf/iotdb-confignode.properties
@@ -212,9 +212,9 @@ target_confignode=0.0.0.0:22277
# proc_wal_dir=data/proc
-# UDF dir
+# UDF lib dir
# If this property is unset, system will save the data in the default relative path directory under
-# the UDF folder(i.e., %CONFIGNODE_HOME%/data/udf).
+# the UDF folder(i.e., %CONFIGNODE_HOME%/ext/udf).
#
# If it is absolute, system will save the data in exact location it points to.
# If it is relative, system will save the data in the relative path directory it indicates under the
@@ -225,11 +225,31 @@ target_confignode=0.0.0.0:22277
# For Window platform
# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is
# absolute. Otherwise, it is relative.
-# udf_dir=data\\udf
+# udf_lib_dir=ext\\udf
#
# For Linux platform
# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
-# udf_dir=data/udf
+# udf_lib_dir=ext/udf
+
+
+# temporary lib dir
+# If this property is unset, system will save the data in the default relative path directory under
+# the UDF folder(i.e., %CONFIGNODE_HOME%/ext/temporary).
+#
+# If it is absolute, system will save the data in exact location it points to.
+# If it is relative, system will save the data in the relative path directory it indicates under the
+# UDF folder.
+# Note: If data_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative
+# path.
+#
+# For Window platform
+# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is
+# absolute. Otherwise, it is relative.
+# temporary_lib_dir=ext\\temporary
+#
+# For Linux platform
+# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
+# temporary_lib_dir=ext/temporary
####################
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConf.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConf.java
index 5362f9f95b..0ca14d6dd5 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConf.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConf.java
@@ -103,9 +103,16 @@ public class ConfigNodeConf {
private String consensusDir =
ConfigNodeConstant.DATA_DIR + File.separator + ConfigNodeConstant.CONSENSUS_FOLDER;
- /** UDF directory, storage udf metadata and jars */
- private String udfDir =
- ConfigNodeConstant.DATA_DIR + File.separator + ConfigNodeConstant.UDF_FOLDER;
+ /** External lib directory, stores user-uploaded JAR files */
+ private String extLibDir = IoTDBConstant.EXT_FOLDER_NAME;
+
+ /** External lib directory for UDF, stores user-uploaded JAR files */
+ private String udfLibDir =
+ IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.UDF_FOLDER_NAME;
+
+ /** External temporary lib directory for storing downloaded JAR files */
+ private String temporaryLibDir =
+ IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.TMP_FOLDER_NAME;
/** Time partition interval in seconds */
private long timePartitionInterval = 604800;
@@ -149,6 +156,9 @@ public class ConfigNodeConf {
private void formulateFolders() {
systemDir = addHomeDir(systemDir);
consensusDir = addHomeDir(consensusDir);
+ extLibDir = addHomeDir(extLibDir);
+ udfLibDir = addHomeDir(udfLibDir);
+ temporaryLibDir = addHomeDir(temporaryLibDir);
}
private String addHomeDir(String dir) {
@@ -336,12 +346,24 @@ public class ConfigNodeConf {
this.systemDir = systemDir;
}
- public String getUdfDir() {
- return udfDir;
+ public String getSystemUdfDir() {
+ return getSystemDir() + File.separator + "udf" + File.separator;
+ }
+
+ public String getUdfLibDir() {
+ return udfLibDir;
+ }
+
+ public void setUdfLibDir(String udfLibDir) {
+ this.udfLibDir = udfLibDir;
+ }
+
+ public String getTemporaryLibDir() {
+ return temporaryLibDir;
}
- public void setUdfDir(String udfDir) {
- this.udfDir = udfDir;
+ public void setTemporaryLibDir(String temporaryLibDir) {
+ this.temporaryLibDir = temporaryLibDir;
}
public int getSchemaReplicationFactor() {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
index b5c58efd92..2a024dd3b0 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
@@ -175,7 +175,10 @@ public class ConfigNodeDescriptor {
conf.setConsensusDir(properties.getProperty("consensus_dir", conf.getConsensusDir()));
- conf.setUdfDir(properties.getProperty("udf_dir", conf.getUdfDir()));
+ conf.setUdfLibDir(properties.getProperty("udf_lib_dir", conf.getUdfLibDir()));
+
+ conf.setTemporaryLibDir(
+ properties.getProperty("temporary_lib_dir", conf.getTemporaryLibDir()));
conf.setTimePartitionInterval(
Long.parseLong(
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java
index b5fbc250aa..e26fedcccd 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java
@@ -20,18 +20,39 @@
package org.apache.iotdb.confignode.manager;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.udf.service.UDFClassLoaderManager;
+import org.apache.iotdb.commons.udf.service.UDFExecutableFetcher;
+import org.apache.iotdb.commons.udf.service.UDFRegistrationService;
+import org.apache.iotdb.confignode.conf.ConfigNodeConf;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
import org.apache.iotdb.confignode.persistence.UDFInfo;
+import java.io.IOException;
import java.util.List;
public class UDFManager {
+ private static final ConfigNodeConf CONFIG_NODE_CONF =
+ ConfigNodeDescriptor.getInstance().getConf();
+
+ private final UDFExecutableFetcher udfExecutableFetcher;
+ private final UDFClassLoaderManager udfClassLoaderManager;
+ private final UDFRegistrationService udfRegistrationService;
+
private final ConfigManager configManager;
private final UDFInfo udfInfo;
- public UDFManager(ConfigManager configManager, UDFInfo udfInfo) {
+ public UDFManager(ConfigManager configManager, UDFInfo udfInfo) throws IOException {
this.configManager = configManager;
this.udfInfo = udfInfo;
+
+ udfExecutableFetcher =
+ UDFExecutableFetcher.setupAndGetInstance(
+ CONFIG_NODE_CONF.getTemporaryLibDir(), CONFIG_NODE_CONF.getUdfLibDir());
+ udfClassLoaderManager =
+ UDFClassLoaderManager.setupAndGetInstance(CONFIG_NODE_CONF.getUdfLibDir());
+ udfRegistrationService =
+ UDFRegistrationService.setupAndGetInstance(CONFIG_NODE_CONF.getSystemUdfDir());
}
public TSStatus createFunction(String udfName, String className, List<String> uris) {
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index 1e033baf64..566ba957c8 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@ -172,6 +172,7 @@ public class IoTDBConstant {
public static final String TRACING_LOG = "tracing.txt";
public static final String EXT_FOLDER_NAME = "ext";
public static final String UDF_FOLDER_NAME = "udf";
+ public static final String TMP_FOLDER_NAME = "temporary";
public static final String TRIGGER_FOLDER_NAME = "trigger";
public static final String MQTT_FOLDER_NAME = "mqtt";
public static final String WAL_FOLDER_NAME = "wal";
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableFetcher.java b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableFetcher.java
new file mode 100644
index 0000000000..d04b1819a2
--- /dev/null
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableFetcher.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.udf.service;
+
+import org.apache.iotdb.commons.file.SystemFileFactory;
+
+import org.apache.commons.io.FileUtils;
+
+import java.io.File;
+import java.io.IOException;
+
+public class UDFExecutableFetcher {
+
+ private final String temporaryLibRoot;
+ private final String extLibRoot;
+
+ private UDFExecutableFetcher(String temporaryLibRoot, String extLibRoot) {
+ this.temporaryLibRoot = temporaryLibRoot;
+ this.extLibRoot = extLibRoot;
+ }
+
+ private static UDFExecutableFetcher INSTANCE = null;
+
+ public static synchronized UDFExecutableFetcher setupAndGetInstance(
+ String temporaryLibRoot, String extLibRoot) throws IOException {
+ if (INSTANCE == null) {
+ makeDirIfNecessary(temporaryLibRoot);
+ makeDirIfNecessary(extLibRoot);
+ INSTANCE = new UDFExecutableFetcher(temporaryLibRoot, extLibRoot);
+ }
+ return INSTANCE;
+ }
+
+ private static void makeDirIfNecessary(String dir) throws IOException {
+ File file = SystemFileFactory.INSTANCE.getFile(dir);
+ if (file.exists() && file.isDirectory()) {
+ return;
+ }
+ FileUtils.forceMkdir(file);
+ }
+
+ public static UDFExecutableFetcher getInstance() {
+ return INSTANCE;
+ }
+}
[iotdb] 06/18: datanode -> confignode
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ae8a2b9882efbc15e9c762cb1e2c8e5314137e6c
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Thu May 19 15:24:40 2022 +0800
datanode -> confignode
---
.../apache/iotdb/db/client/ConfigNodeClient.java | 17 ++++
.../plan/execution/config/CreateFunctionTask.java | 100 ++++++++++++---------
.../src/main/thrift/confignode.thrift | 4 +-
3 files changed, 76 insertions(+), 45 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java b/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java
index f1494eeb85..991d71bc96 100644
--- a/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java
+++ b/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java
@@ -36,6 +36,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp;
import org.apache.iotdb.confignode.rpc.thrift.TCountStorageGroupResp;
+import org.apache.iotdb.confignode.rpc.thrift.TCreateFuctionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfoResp;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterResp;
@@ -568,6 +569,22 @@ public class ConfigNodeClient implements ConfigIService.Iface, SyncThriftClient,
throw new TException(MSG_RECONNECTION_FAIL);
}
+ @Override
+ public TSStatus createFunction(TCreateFuctionReq req) throws TException {
+ for (int i = 0; i < RETRY_NUM; i++) {
+ try {
+ TSStatus status = client.createFunction(req);
+ if (!updateConfigNodeLeader(status)) {
+ return status;
+ }
+ } catch (TException e) {
+ configLeader = null;
+ }
+ reconnect();
+ }
+ throw new TException(MSG_RECONNECTION_FAIL);
+ }
+
public static class Factory extends BaseClientFactory<PartitionRegionId, ConfigNodeClient> {
public Factory(
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/CreateFunctionTask.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/CreateFunctionTask.java
index 663f533729..e251015cd8 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/CreateFunctionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/CreateFunctionTask.java
@@ -19,17 +19,29 @@
package org.apache.iotdb.db.mpp.plan.execution.config;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.client.IClientManager;
import org.apache.iotdb.commons.consensus.PartitionRegionId;
+import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq;
import org.apache.iotdb.db.client.ConfigNodeClient;
+import org.apache.iotdb.db.client.ConfigNodeInfo;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateFunctionStatement;
+import org.apache.iotdb.rpc.StatementExecutionException;
+import org.apache.iotdb.rpc.TSStatusCode;
import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.stream.Collectors;
+
public class CreateFunctionTask implements IConfigTask {
private static final Logger LOGGER = LoggerFactory.getLogger(CreateFunctionTask.class);
@@ -45,48 +57,50 @@ public class CreateFunctionTask implements IConfigTask {
public ListenableFuture<ConfigTaskResult> execute(
IClientManager<PartitionRegionId, ConfigNodeClient> clientManager)
throws InterruptedException {
- return null;
- // SettableFuture<ConfigTaskResult> future = SettableFuture.create();
- // if (CONFIG.isClusterMode()) {
- // TDeleteStorageGroupsReq req =
- // new TDeleteStorageGroupsReq(deleteStorageGroupStatement.getPrefixPath());
- // try (ConfigNodeClient client =
- // clientManager.borrowClient(ConfigNodeInfo.partitionRegionId)) {
- // TSStatus tsStatus = client.deleteStorageGroups(req);
- // if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) {
- // LOGGER.error(
- // "Failed to execute delete storage group {} in config node, status is {}.",
- // deleteStorageGroupStatement.getPrefixPath(),
- // tsStatus);
- // future.setException(new StatementExecutionException(tsStatus));
- // } else {
- // future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS));
- // }
- // } catch (TException | IOException e) {
- // LOGGER.error("Failed to connect to config node.");
- // future.setException(e);
- // }
- // } else {
- // try {
- // List<PartialPath> deletePathList =
- // deleteStorageGroupStatement.getPrefixPath().stream()
- // .map(
- // path -> {
- // try {
- // return new PartialPath(path);
- // } catch (IllegalPathException e) {
- // return null;
- // }
- // })
- // .collect(Collectors.toList());
- // LocalConfigNode.getInstance().deleteStorageGroups(deletePathList);
- // } catch (MetadataException e) {
- // future.setException(e);
- // }
- // future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS));
- // }
- // // If the action is executed successfully, return the Future.
- // // If your operation is async, you can return the corresponding future directly.
- // return future;
+ SettableFuture<ConfigTaskResult> future = SettableFuture.create();
+ if (CONFIG.isClusterMode()) {
+ executeCluster(clientManager, future);
+ } else {
+ executeStandalone(clientManager, future);
+ }
+ return future;
+ }
+
+ private void executeCluster(
+ IClientManager<PartitionRegionId, ConfigNodeClient> clientManager,
+ SettableFuture<ConfigTaskResult> future) {
+ try (ConfigNodeClient client = clientManager.borrowClient(ConfigNodeInfo.partitionRegionId)) {
+ final String udfName = createFunctionStatement.getUdfName();
+ final String className = createFunctionStatement.getClassName();
+ final List<String> uris =
+ createFunctionStatement.getUris().stream()
+ .map(URI::toString)
+ .collect(Collectors.toList());
+
+ final TSStatus executionStatus =
+ client.createFunction(new TCreateFunctionReq(udfName, className, uris));
+
+ if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != executionStatus.getCode()) {
+ LOGGER.error(
+ "[{}] Failed to create function {}({}) in config node, URI: {}.",
+ executionStatus,
+ udfName,
+ className,
+ uris);
+ future.setException(new StatementExecutionException(executionStatus));
+ } else {
+ future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS));
+ }
+ } catch (TException | IOException e) {
+ LOGGER.error("Failed to connect to config node.");
+ future.setException(e);
+ }
+ }
+
+ private void executeStandalone(
+ IClientManager<PartitionRegionId, ConfigNodeClient> clientManager,
+ SettableFuture<ConfigTaskResult> future) {
+ // TODO
+ throw new UnsupportedOperationException();
}
}
diff --git a/thrift-confignode/src/main/thrift/confignode.thrift b/thrift-confignode/src/main/thrift/confignode.thrift
index af0e8fada9..20b325791d 100644
--- a/thrift-confignode/src/main/thrift/confignode.thrift
+++ b/thrift-confignode/src/main/thrift/confignode.thrift
@@ -175,7 +175,7 @@ struct TConfigNodeRegisterResp {
}
// UDF
-struct TCreateFuctionReq {
+struct TCreateFunctionReq {
1: required string udfName
2: required string className
3: required list<string> uris
@@ -239,5 +239,5 @@ service ConfigIService {
/* UDF */
- common.TSStatus createFunction(TCreateFuctionReq req)
+ common.TSStatus createFunction(TCreateFunctionReq req)
}
\ No newline at end of file
[iotdb] 04/18: visit task
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b2fe9293ab256f321f9ad576c0825636c3a05afa
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Thu May 19 10:59:54 2022 +0800
visit task
---
.../plan/execution/config/ConfigTaskVisitor.java | 8 ++++
.../plan/execution/config/CreateFunctionTask.java | 43 ++++++++++++++++++++++
2 files changed, 51 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ConfigTaskVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ConfigTaskVisitor.java
index 671edfb68b..ebd9327e71 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ConfigTaskVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ConfigTaskVisitor.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.db.mpp.plan.statement.Statement;
import org.apache.iotdb.db.mpp.plan.statement.StatementNode;
import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CountStorageGroupStatement;
+import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateFunctionStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.DeleteStorageGroupStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.SetStorageGroupStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.SetTTLStatement;
@@ -51,6 +52,7 @@ public class ConfigTaskVisitor
return new SetStorageGroupTask(statement);
}
+ @Override
public IConfigTask visitDeleteStorageGroup(
DeleteStorageGroupStatement statement, TaskContext context) {
return new DeleteStorageGroupTask(statement);
@@ -88,5 +90,11 @@ public class ConfigTaskVisitor
return new AuthorizerConfigTask(statement);
}
+ @Override
+ public IConfigTask visitCreateFunction(
+ CreateFunctionStatement createFunctionStatement, TaskContext context) {
+ return new CreateFunctionTask(createFunctionStatement);
+ }
+
public static class TaskContext {}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/CreateFunctionTask.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/CreateFunctionTask.java
new file mode 100644
index 0000000000..62cfc5f965
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/CreateFunctionTask.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.plan.execution.config;
+
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.consensus.PartitionRegionId;
+import org.apache.iotdb.db.client.ConfigNodeClient;
+import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateFunctionStatement;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+public class CreateFunctionTask implements IConfigTask {
+
+ private final CreateFunctionStatement createFunctionStatement;
+
+ public CreateFunctionTask(CreateFunctionStatement createFunctionStatement) {
+ this.createFunctionStatement = createFunctionStatement;
+ }
+
+ @Override
+ public ListenableFuture<ConfigTaskResult> execute(
+ IClientManager<PartitionRegionId, ConfigNodeClient> clientManager)
+ throws InterruptedException {
+ return null;
+ }
+}
[iotdb] 18/18: register function in UDFInfo
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c9355e5d542f5dc7e9d72647ee033d9dcef49cd0
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Tue May 24 01:37:54 2022 +0800
register function in UDFInfo
---
.../consensus/request/write/CreateFunctionReq.java | 50 ++++++-
.../iotdb/confignode/manager/UDFManager.java | 72 +++++++---
.../iotdb/confignode/persistence/UDFInfo.java | 41 +++++-
.../commons/udf/service/UDFExecutableFetcher.java | 62 ---------
.../commons/udf/service/UDFExecutableManager.java | 146 +++++++++++++++++++++
.../commons/udf/service/UDFExecutableResource.java | 26 ++--
.../iotdb/commons/udf/service/UDFLogWriter.java | 21 ++-
.../udf/service/UDFRegistrationInformation.java | 13 +-
.../udf/service/UDFRegistrationService.java | 78 +++++++++--
9 files changed, 399 insertions(+), 110 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/CreateFunctionReq.java b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/CreateFunctionReq.java
index 2cfe6ffe44..4cc034a335 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/CreateFunctionReq.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/CreateFunctionReq.java
@@ -21,19 +21,65 @@ package org.apache.iotdb.confignode.consensus.request.write;
import org.apache.iotdb.confignode.consensus.request.ConfigRequest;
import org.apache.iotdb.confignode.consensus.request.ConfigRequestType;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
public class CreateFunctionReq extends ConfigRequest {
+ private String functionName;
+ private String className;
+ private List<String> uris;
+
public CreateFunctionReq() {
super(ConfigRequestType.CreateFunction);
}
+ public CreateFunctionReq(String functionName, String className, List<String> uris) {
+ super(ConfigRequestType.CreateFunction);
+ this.functionName = functionName;
+ this.className = className;
+ this.uris = uris;
+ }
+
+ public String getFunctionName() {
+ return functionName;
+ }
+
+ public String getClassName() {
+ return className;
+ }
+
+ public List<String> getUris() {
+ return uris;
+ }
+
@Override
- protected void serializeImpl(ByteBuffer buffer) {}
+ protected void serializeImpl(ByteBuffer buffer) {
+ buffer.putInt(getType().ordinal());
+
+ ReadWriteIOUtils.write(functionName, buffer);
+ ReadWriteIOUtils.write(className, buffer);
+
+ final int size = uris.size();
+ ReadWriteIOUtils.write(size, buffer);
+ for (String uri : uris) {
+ ReadWriteIOUtils.write(uri, buffer);
+ }
+ }
@Override
- protected void deserializeImpl(ByteBuffer buffer) throws IOException {}
+ protected void deserializeImpl(ByteBuffer buffer) throws IOException {
+ functionName = ReadWriteIOUtils.readString(buffer);
+ className = ReadWriteIOUtils.readString(buffer);
+
+ final int size = ReadWriteIOUtils.readInt(buffer);
+ uris = new ArrayList<>(size);
+ for (int i = 0; i < size; ++i) {
+ uris.add(ReadWriteIOUtils.readString(buffer));
+ }
+ }
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java
index e26fedcccd..f2ee00745d 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java
@@ -20,42 +20,82 @@
package org.apache.iotdb.confignode.manager;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.udf.service.UDFClassLoaderManager;
-import org.apache.iotdb.commons.udf.service.UDFExecutableFetcher;
-import org.apache.iotdb.commons.udf.service.UDFRegistrationService;
+import org.apache.iotdb.commons.udf.service.UDFClassLoader;
+import org.apache.iotdb.commons.udf.service.UDFExecutableManager;
+import org.apache.iotdb.commons.udf.service.UDFExecutableResource;
import org.apache.iotdb.confignode.conf.ConfigNodeConf;
import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
+import org.apache.iotdb.confignode.consensus.request.write.CreateFunctionReq;
import org.apache.iotdb.confignode.persistence.UDFInfo;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
public class UDFManager {
+ private static final Logger LOGGER = LoggerFactory.getLogger(UDFManager.class);
+
private static final ConfigNodeConf CONFIG_NODE_CONF =
ConfigNodeDescriptor.getInstance().getConf();
- private final UDFExecutableFetcher udfExecutableFetcher;
- private final UDFClassLoaderManager udfClassLoaderManager;
- private final UDFRegistrationService udfRegistrationService;
-
private final ConfigManager configManager;
private final UDFInfo udfInfo;
+ private final UDFExecutableManager udfExecutableManager;
public UDFManager(ConfigManager configManager, UDFInfo udfInfo) throws IOException {
this.configManager = configManager;
this.udfInfo = udfInfo;
-
- udfExecutableFetcher =
- UDFExecutableFetcher.setupAndGetInstance(
+ udfExecutableManager =
+ UDFExecutableManager.setupAndGetInstance(
CONFIG_NODE_CONF.getTemporaryLibDir(), CONFIG_NODE_CONF.getUdfLibDir());
- udfClassLoaderManager =
- UDFClassLoaderManager.setupAndGetInstance(CONFIG_NODE_CONF.getUdfLibDir());
- udfRegistrationService =
- UDFRegistrationService.setupAndGetInstance(CONFIG_NODE_CONF.getSystemUdfDir());
}
- public TSStatus createFunction(String udfName, String className, List<String> uris) {
- return new TSStatus();
+ public TSStatus createFunction(String functionName, String className, List<String> uris) {
+ try {
+ if (uris.isEmpty()) {
+ fetchExecutablesAndCheckInstantiation(className);
+ } else {
+ fetchExecutablesAndCheckInstantiation(className, uris);
+ }
+
+ // TODO: notify data nodes
+ return configManager
+ .getConsensusManager()
+ .write(new CreateFunctionReq(functionName, className, uris))
+ .getStatus();
+ } catch (Exception e) {
+ final String errorMessage =
+ String.format(
+ "Failed to register UDF %s(class name: %s, uris: %s), because of exception: %s",
+ functionName, className, uris, e);
+ LOGGER.warn(errorMessage);
+ return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())
+ .setMessage(errorMessage);
+ }
+ }
+
+ private void fetchExecutablesAndCheckInstantiation(String className) throws Exception {
+ try (UDFClassLoader temporaryUdfClassLoader =
+ new UDFClassLoader(CONFIG_NODE_CONF.getUdfLibDir())) {
+ Class.forName(className, true, temporaryUdfClassLoader)
+ .getDeclaredConstructor()
+ .newInstance();
+ }
+ }
+
+ private void fetchExecutablesAndCheckInstantiation(String className, List<String> uris)
+ throws Exception {
+ final UDFExecutableResource resource = udfExecutableManager.request(uris);
+ try (UDFClassLoader temporaryUdfClassLoader = new UDFClassLoader(resource.getResourceDir())) {
+ Class.forName(className, true, temporaryUdfClassLoader)
+ .getDeclaredConstructor()
+ .newInstance();
+ } finally {
+ udfExecutableManager.removeFromTemporaryLibRoot(resource);
+ }
}
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/UDFInfo.java b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/UDFInfo.java
index 4ba470d7c9..5c3a61d25e 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/UDFInfo.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/UDFInfo.java
@@ -21,17 +21,56 @@ package org.apache.iotdb.confignode.persistence;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.snapshot.SnapshotProcessor;
+import org.apache.iotdb.commons.udf.service.UDFExecutableManager;
+import org.apache.iotdb.commons.udf.service.UDFRegistrationService;
+import org.apache.iotdb.confignode.conf.ConfigNodeConf;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
import org.apache.iotdb.confignode.consensus.request.write.CreateFunctionReq;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.List;
public class UDFInfo implements SnapshotProcessor {
+ private static final Logger LOGGER = LoggerFactory.getLogger(UDFInfo.class);
+
+ private static final ConfigNodeConf CONFIG_NODE_CONF =
+ ConfigNodeDescriptor.getInstance().getConf();
+
+ private final UDFExecutableManager udfExecutableManager;
+ private final UDFRegistrationService udfRegistrationService;
+
+ public UDFInfo() throws IOException {
+ udfExecutableManager =
+ UDFExecutableManager.setupAndGetInstance(
+ CONFIG_NODE_CONF.getTemporaryLibDir(), CONFIG_NODE_CONF.getUdfLibDir());
+ udfRegistrationService =
+ UDFRegistrationService.setupAndGetInstance(CONFIG_NODE_CONF.getSystemUdfDir());
+ }
+
public TSStatus createFunction(CreateFunctionReq req) {
- throw new UnsupportedOperationException();
+ final String functionName = req.getFunctionName();
+ final String className = req.getClassName();
+ final List<String> uris = req.getUris();
+
+ try {
+ udfRegistrationService.register(functionName, className, uris, udfExecutableManager, true);
+ return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ } catch (Exception e) {
+ final String errorMessage =
+ String.format(
+ "Failed to register UDF %s(class name: %s, uris: %s), because of exception: %s",
+ functionName, className, uris, e);
+ LOGGER.warn(errorMessage);
+ return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())
+ .setMessage(errorMessage);
+ }
}
@Override
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableFetcher.java b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableFetcher.java
deleted file mode 100644
index d04b1819a2..0000000000
--- a/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableFetcher.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.commons.udf.service;
-
-import org.apache.iotdb.commons.file.SystemFileFactory;
-
-import org.apache.commons.io.FileUtils;
-
-import java.io.File;
-import java.io.IOException;
-
-public class UDFExecutableFetcher {
-
- private final String temporaryLibRoot;
- private final String extLibRoot;
-
- private UDFExecutableFetcher(String temporaryLibRoot, String extLibRoot) {
- this.temporaryLibRoot = temporaryLibRoot;
- this.extLibRoot = extLibRoot;
- }
-
- private static UDFExecutableFetcher INSTANCE = null;
-
- public static synchronized UDFExecutableFetcher setupAndGetInstance(
- String temporaryLibRoot, String extLibRoot) throws IOException {
- if (INSTANCE == null) {
- makeDirIfNecessary(temporaryLibRoot);
- makeDirIfNecessary(extLibRoot);
- INSTANCE = new UDFExecutableFetcher(temporaryLibRoot, extLibRoot);
- }
- return INSTANCE;
- }
-
- private static void makeDirIfNecessary(String dir) throws IOException {
- File file = SystemFileFactory.INSTANCE.getFile(dir);
- if (file.exists() && file.isDirectory()) {
- return;
- }
- FileUtils.forceMkdir(file);
- }
-
- public static UDFExecutableFetcher getInstance() {
- return INSTANCE;
- }
-}
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableManager.java b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableManager.java
new file mode 100644
index 0000000000..5f03dbe04b
--- /dev/null
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableManager.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.udf.service;
+
+import org.apache.iotdb.commons.file.SystemFileFactory;
+import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+
+import org.apache.commons.io.FileUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class UDFExecutableManager {
+
+ private final String temporaryLibRoot;
+ private final String extLibRoot;
+
+ private final AtomicLong requestCounter;
+
+ private UDFExecutableManager(String temporaryLibRoot, String extLibRoot) {
+ this.temporaryLibRoot = temporaryLibRoot;
+ this.extLibRoot = extLibRoot;
+
+ requestCounter = new AtomicLong(0);
+ }
+
+ public UDFExecutableResource request(List<String> uris) throws URISyntaxException, IOException {
+ final long requestId = generateNextRequestId();
+ downloadExecutables(uris, requestId);
+ return new UDFExecutableResource(requestId, getDirStringByRequestId(requestId));
+ }
+
+ public void moveToExtLibDir(UDFExecutableResource resource, String functionName)
+ throws IOException {
+ FileUtils.moveDirectory(
+ getDirByRequestId(resource.getRequestId()), getDirByFunctionName(functionName));
+ }
+
+ public void removeFromTemporaryLibRoot(UDFExecutableResource resource) throws IOException {
+ removeFromTemporaryLibRoot(resource.getRequestId());
+ }
+
+ public void removeFromExtLibDir(String functionName) {
+ FileUtils.deleteQuietly(getDirByFunctionName(functionName));
+ }
+
+ private synchronized long generateNextRequestId() throws IOException {
+ long requestId = requestCounter.getAndIncrement();
+ while (FileUtils.isDirectory(getDirByRequestId(requestId))) {
+ requestId = requestCounter.getAndIncrement();
+ }
+ FileUtils.forceMkdir(getDirByRequestId(requestId));
+ return requestId;
+ }
+
+ private void downloadExecutables(List<String> uris, long requestId)
+ throws IOException, URISyntaxException {
+ // TODO: para download
+ try {
+ for (String uriString : uris) {
+ final URL url = new URI(uriString).toURL();
+ final String fileName = uriString.substring(uriString.lastIndexOf("/") + 1);
+ final String destination =
+ temporaryLibRoot + File.separator + requestId + File.separator + fileName;
+ FileUtils.copyURLToFile(url, FSFactoryProducer.getFSFactory().getFile(destination));
+ }
+ } catch (Exception e) {
+ removeFromTemporaryLibRoot(requestId);
+ throw e;
+ }
+ }
+
+ private void removeFromTemporaryLibRoot(long requestId) {
+ FileUtils.deleteQuietly(getDirByRequestId(requestId));
+ }
+
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // dir string and dir file generation
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+
+ public File getDirByRequestId(long requestId) {
+ return FSFactoryProducer.getFSFactory().getFile(getDirStringByRequestId(requestId));
+ }
+
+ public String getDirStringByRequestId(long requestId) {
+ return temporaryLibRoot + File.separator + requestId + File.separator;
+ }
+
+ public File getDirByFunctionName(String functionName) {
+ return FSFactoryProducer.getFSFactory().getFile(getDirStringByFunctionName(functionName));
+ }
+
+ public String getDirStringByFunctionName(String functionName) {
+ return extLibRoot + File.separator + functionName + File.separator;
+ }
+
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // singleton instance holder
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+
+ private static UDFExecutableManager INSTANCE = null;
+
+ public static synchronized UDFExecutableManager setupAndGetInstance(
+ String temporaryLibRoot, String extLibRoot) throws IOException {
+ if (INSTANCE == null) {
+ makeDirIfNecessary(temporaryLibRoot);
+ makeDirIfNecessary(extLibRoot);
+ INSTANCE = new UDFExecutableManager(temporaryLibRoot, extLibRoot);
+ }
+ return INSTANCE;
+ }
+
+ private static void makeDirIfNecessary(String dir) throws IOException {
+ File file = SystemFileFactory.INSTANCE.getFile(dir);
+ if (file.exists() && file.isDirectory()) {
+ return;
+ }
+ FileUtils.forceMkdir(file);
+ }
+
+ public static UDFExecutableManager getInstance() {
+ return INSTANCE;
+ }
+}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/CreateFunctionReq.java b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableResource.java
similarity index 60%
copy from confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/CreateFunctionReq.java
copy to node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableResource.java
index 2cfe6ffe44..ec0c375997 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/CreateFunctionReq.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFExecutableResource.java
@@ -17,23 +17,23 @@
* under the License.
*/
-package org.apache.iotdb.confignode.consensus.request.write;
+package org.apache.iotdb.commons.udf.service;
-import org.apache.iotdb.confignode.consensus.request.ConfigRequest;
-import org.apache.iotdb.confignode.consensus.request.ConfigRequestType;
+public class UDFExecutableResource {
-import java.io.IOException;
-import java.nio.ByteBuffer;
+ private final long requestId;
+ private final String resourceDir;
-public class CreateFunctionReq extends ConfigRequest {
-
- public CreateFunctionReq() {
- super(ConfigRequestType.CreateFunction);
+ public UDFExecutableResource(long requestId, String resourceDir) {
+ this.requestId = requestId;
+ this.resourceDir = resourceDir;
}
- @Override
- protected void serializeImpl(ByteBuffer buffer) {}
+ public long getRequestId() {
+ return requestId;
+ }
- @Override
- protected void deserializeImpl(ByteBuffer buffer) throws IOException {}
+ public String getResourceDir() {
+ return resourceDir;
+ }
}
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFLogWriter.java b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFLogWriter.java
index 51068cf99b..54e8bde4a0 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFLogWriter.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFLogWriter.java
@@ -25,14 +25,18 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.util.List;
public class UDFLogWriter {
- public static final Byte REGISTER_TYPE = 0;
+ public static final Byte REGISTER_WITHOUT_URIS_TYPE = 0;
public static final Byte DEREGISTER_TYPE = 1;
+ public static final Byte REGISTER_WITH_URIS_TYPE = 2;
- private static final String REGISTER_TYPE_STRING = REGISTER_TYPE.toString();
+ private static final String REGISTER_WITHOUT_URIS_TYPE_STRING =
+ REGISTER_WITHOUT_URIS_TYPE.toString();
private static final String DEREGISTER_TYPE_STRING = DEREGISTER_TYPE.toString();
+ private static final String REGISTER_WITH_URIS_TYPE_STRING = REGISTER_WITH_URIS_TYPE.toString();
private final File logFile;
private final BufferedWriter writer;
@@ -53,8 +57,17 @@ public class UDFLogWriter {
}
}
- public void register(String functionName, String className) throws IOException {
- writer.write(String.format("%s,%s,%s", REGISTER_TYPE_STRING, functionName, className));
+ public void register(String functionName, String className, List<String> uris)
+ throws IOException {
+ if (uris.isEmpty()) {
+ writer.write(
+ String.format("%s,%s,%s", REGISTER_WITHOUT_URIS_TYPE_STRING, functionName, className));
+ } else {
+ writer.write(
+ String.format(
+ "%s,%s,%s,%s",
+ REGISTER_WITH_URIS_TYPE_STRING, functionName, className, String.join(",", uris)));
+ }
writeLineAndFlush();
}
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFRegistrationInformation.java b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFRegistrationInformation.java
index a4997ead0b..feb799f2db 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFRegistrationInformation.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFRegistrationInformation.java
@@ -22,19 +22,26 @@ package org.apache.iotdb.commons.udf.service;
import org.apache.iotdb.commons.udf.api.UDTF;
import java.lang.reflect.InvocationTargetException;
+import java.util.List;
public class UDFRegistrationInformation {
private final String functionName;
private final String className;
+ private final List<String> uris;
private final boolean isBuiltin;
private Class<?> functionClass;
public UDFRegistrationInformation(
- String functionName, String className, boolean isBuiltin, Class<?> functionClass) {
+ String functionName,
+ String className,
+ List<String> uris,
+ boolean isBuiltin,
+ Class<?> functionClass) {
this.functionName = functionName;
this.className = className;
+ this.uris = uris;
this.isBuiltin = isBuiltin;
this.functionClass = functionClass;
}
@@ -47,6 +54,10 @@ public class UDFRegistrationInformation {
return className;
}
+ public List<String> getUris() {
+ return uris;
+ }
+
public boolean isBuiltin() {
return isBuiltin;
}
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFRegistrationService.java b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFRegistrationService.java
index 3f9be546d1..2d81f4b710 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFRegistrationService.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/udf/service/UDFRegistrationService.java
@@ -31,6 +31,7 @@ import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.Validate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,8 +39,11 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
import java.lang.reflect.InvocationTargetException;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantLock;
@@ -82,8 +86,25 @@ public class UDFRegistrationService implements IService {
functionName = functionName.toUpperCase();
validateFunctionName(functionName, className);
checkIfRegistered(functionName, className);
- doRegister(functionName, className);
- tryAppendRegistrationLog(functionName, className, writeToTemporaryLogFile);
+ doRegister(functionName, className, Collections.emptyList());
+ tryAppendRegistrationLog(
+ functionName, className, Collections.emptyList(), writeToTemporaryLogFile);
+ }
+
+ public void register(
+ String functionName,
+ String className,
+ List<String> uris,
+ UDFExecutableManager udfExecutableManager,
+ boolean writeToTemporaryLogFile)
+ throws UDFRegistrationException {
+ Validate.isTrue(uris != null && !uris.isEmpty());
+ functionName = functionName.toUpperCase();
+ validateFunctionName(functionName, className);
+ checkIfRegistered(functionName, className);
+ downloadExecutableResources(functionName, className, uris, udfExecutableManager);
+ doRegister(functionName, className, uris);
+ tryAppendRegistrationLog(functionName, className, uris, writeToTemporaryLogFile);
}
private static void validateFunctionName(String functionName, String className)
@@ -132,7 +153,34 @@ public class UDFRegistrationService implements IService {
throw new UDFRegistrationException(errorMessage);
}
- private void doRegister(String functionName, String className) throws UDFRegistrationException {
+ private void downloadExecutableResources(
+ String functionName,
+ String className,
+ List<String> uris,
+ UDFExecutableManager udfExecutableManager)
+ throws UDFRegistrationException {
+ try {
+ final UDFExecutableResource resource = udfExecutableManager.request(uris);
+ try {
+ udfExecutableManager.removeFromExtLibDir(functionName);
+ udfExecutableManager.moveToExtLibDir(resource, functionName);
+ } catch (Exception innerException) {
+ udfExecutableManager.removeFromExtLibDir(functionName);
+ udfExecutableManager.removeFromTemporaryLibRoot(resource);
+ throw innerException;
+ }
+ } catch (Exception outerException) {
+ String errorMessage =
+ String.format(
+ "Failed to register UDF %s(%s) because failed to fetch UDF executables(%s)",
+ functionName, className, uris);
+ LOGGER.warn(errorMessage);
+ throw new UDFRegistrationException(errorMessage, outerException);
+ }
+ }
+
+ private void doRegister(String functionName, String className, List<String> uris)
+ throws UDFRegistrationException {
acquireRegistrationLock();
try {
UDFClassLoader currentActiveClassLoader =
@@ -143,7 +191,7 @@ public class UDFRegistrationService implements IService {
functionClass.getDeclaredConstructor().newInstance();
registrationInformation.put(
functionName,
- new UDFRegistrationInformation(functionName, className, false, functionClass));
+ new UDFRegistrationInformation(functionName, className, uris, false, functionClass));
} catch (IOException
| InstantiationException
| InvocationTargetException
@@ -162,14 +210,14 @@ public class UDFRegistrationService implements IService {
}
private void tryAppendRegistrationLog(
- String functionName, String className, boolean writeToTemporaryLogFile)
+ String functionName, String className, List<String> uris, boolean writeToTemporaryLogFile)
throws UDFRegistrationException {
if (!writeToTemporaryLogFile) {
return;
}
try {
- appendRegistrationLog(functionName, className);
+ appendRegistrationLog(functionName, className, uris);
} catch (IOException e) {
registrationInformation.remove(functionName);
String errorMessage =
@@ -218,10 +266,11 @@ public class UDFRegistrationService implements IService {
}
}
- private void appendRegistrationLog(String functionName, String className) throws IOException {
+ private void appendRegistrationLog(String functionName, String className, List<String> uris)
+ throws IOException {
logWriterLock.writeLock().lock();
try {
- logWriter.register(functionName, className);
+ logWriter.register(functionName, className, uris);
} finally {
logWriterLock.writeLock().unlock();
}
@@ -293,6 +342,7 @@ public class UDFRegistrationService implements IService {
new UDFRegistrationInformation(
functionName,
builtinTimeSeriesGeneratingFunction.getClassName(),
+ Collections.emptyList(),
true,
builtinTimeSeriesGeneratingFunction.getFunctionClass()));
}
@@ -331,10 +381,13 @@ public class UDFRegistrationService implements IService {
while ((line = reader.readLine()) != null) {
String[] data = line.split(",");
byte type = Byte.parseByte(data[0]);
- if (type == UDFLogWriter.REGISTER_TYPE) {
+ if (type == UDFLogWriter.REGISTER_WITHOUT_URIS_TYPE
+ || type == UDFLogWriter.REGISTER_WITH_URIS_TYPE) {
recoveredUDFs.put(data[1], data[2]);
} else if (type == UDFLogWriter.DEREGISTER_TYPE) {
recoveredUDFs.remove(data[1]);
+ } else {
+ throw new UnsupportedEncodingException();
}
}
}
@@ -370,7 +423,8 @@ public class UDFRegistrationService implements IService {
if (information.isBuiltin()) {
continue;
}
- temporaryLogFile.register(information.getFunctionName(), information.getClassName());
+ temporaryLogFile.register(
+ information.getFunctionName(), information.getClassName(), information.getUris());
}
temporaryLogFile.close();
}
@@ -391,7 +445,9 @@ public class UDFRegistrationService implements IService {
Class<?> functionClass = Class.forName(className, true, classLoader);
functionName = functionName.toUpperCase();
registrationInformation.put(
- functionName, new UDFRegistrationInformation(functionName, className, true, functionClass));
+ functionName,
+ new UDFRegistrationInformation(
+ functionName, className, Collections.emptyList(), true, functionClass));
}
@TestOnly
[iotdb] 14/18: Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 966a6d30d0644dbb628a02d319ad30899bf715ff
Merge: 00e6a63378 c053a2a766
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Mon May 23 10:45:17 2022 +0800
Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 6 +-
.../query/ClusterPhysicalGeneratorTest.java | 4 +-
.../query/last/ClusterLastQueryExecutorTest.java | 2 +-
.../request/read/GetChildNodesPartitionReq.java | 3 +-
.../request/read/GetChildPathsPartitionReq.java | 3 +-
.../confignode/persistence/PartitionInfo.java | 6 +-
.../java/org/apache/iotdb/udf/UDTFExample.java | 14 +-
.../iotdb/db/query/udf/example/Accumulator.java | 22 +-
.../apache/iotdb/db/query/udf/example/Adder.java | 14 +-
.../apache/iotdb/db/query/udf/example/Counter.java | 18 +-
.../org/apache/iotdb/db/query/udf/example/Max.java | 14 +-
.../iotdb/db/query/udf/example/Multiplier.java | 14 +-
.../SlidingSizeWindowConstructorTester0.java | 12 +-
.../SlidingSizeWindowConstructorTester1.java | 14 +-
.../SlidingTimeWindowConstructionTester.java | 16 +-
.../db/query/udf/example/TerminateTester.java | 12 +-
.../iotdb/db/query/udf/example/ValidateTester.java | 10 +-
.../iotdb/db/query/udf/example/WindowStartEnd.java | 14 +-
.../iotdb/db/integration/IoTDBFuzzyQueryIT.java | 4 +-
.../IoTDBSyntaxConventionStringLiteralIT.java | 14 +-
.../iotdb/db/integration/IoTDBUDFManagementIT.java | 2 +-
.../db/integration/IoTDBUDTFBuiltinFunctionIT.java | 2 +-
.../session/IoTDBSessionSyntaxConventionIT.java | 237 +++++++++++++++++++++
library-udf/pom.xml | 2 +-
.../org/apache/iotdb/library/anomaly/UDTFIQR.java | 14 +-
.../apache/iotdb/library/anomaly/UDTFKSigma.java | 14 +-
.../org/apache/iotdb/library/anomaly/UDTFLOF.java | 14 +-
.../iotdb/library/anomaly/UDTFMissDetect.java | 14 +-
.../apache/iotdb/library/anomaly/UDTFRange.java | 14 +-
.../iotdb/library/anomaly/UDTFTwoSidedFilter.java | 14 +-
.../iotdb/library/anomaly/util/MissDetector.java | 4 +-
.../iotdb/library/anomaly/util/WindowDetect.java | 4 +-
.../org/apache/iotdb/library/dmatch/UDAFCov.java | 14 +-
.../org/apache/iotdb/library/dmatch/UDAFDtw.java | 16 +-
.../apache/iotdb/library/dmatch/UDAFPearson.java | 14 +-
.../apache/iotdb/library/dmatch/UDTFPtnSym.java | 16 +-
.../org/apache/iotdb/library/dmatch/UDTFXCorr.java | 14 +-
.../iotdb/library/dprofile/UDAFIntegral.java | 14 +-
.../iotdb/library/dprofile/UDAFIntegralAvg.java | 14 +-
.../org/apache/iotdb/library/dprofile/UDAFMad.java | 14 +-
.../apache/iotdb/library/dprofile/UDAFMedian.java | 14 +-
.../apache/iotdb/library/dprofile/UDAFMode.java | 14 +-
.../iotdb/library/dprofile/UDAFPercentile.java | 14 +-
.../apache/iotdb/library/dprofile/UDAFPeriod.java | 18 +-
.../apache/iotdb/library/dprofile/UDAFSkew.java | 14 +-
.../apache/iotdb/library/dprofile/UDAFSpread.java | 14 +-
.../apache/iotdb/library/dprofile/UDAFStddev.java | 14 +-
.../org/apache/iotdb/library/dprofile/UDTFACF.java | 14 +-
.../iotdb/library/dprofile/UDTFDistinct.java | 14 +-
.../iotdb/library/dprofile/UDTFHistogram.java | 14 +-
.../apache/iotdb/library/dprofile/UDTFMinMax.java | 14 +-
.../apache/iotdb/library/dprofile/UDTFMvAvg.java | 14 +-
.../apache/iotdb/library/dprofile/UDTFPACF.java | 14 +-
.../org/apache/iotdb/library/dprofile/UDTFQLB.java | 14 +-
.../iotdb/library/dprofile/UDTFResample.java | 14 +-
.../apache/iotdb/library/dprofile/UDTFSample.java | 20 +-
.../apache/iotdb/library/dprofile/UDTFSegment.java | 14 +-
.../apache/iotdb/library/dprofile/UDTFSpline.java | 14 +-
.../apache/iotdb/library/dprofile/UDTFZScore.java | 14 +-
.../dprofile/util/ExactOrderStatistics.java | 4 +-
.../iotdb/library/dquality/UDTFCompleteness.java | 14 +-
.../iotdb/library/dquality/UDTFConsistency.java | 14 +-
.../iotdb/library/dquality/UDTFTimeliness.java | 14 +-
.../iotdb/library/dquality/UDTFValidity.java | 14 +-
.../library/dquality/util/TimeSeriesQuality.java | 4 +-
.../iotdb/library/drepair/UDTFTimestampRepair.java | 14 +-
.../iotdb/library/drepair/UDTFValueFill.java | 14 +-
.../iotdb/library/drepair/UDTFValueRepair.java | 14 +-
.../apache/iotdb/library/drepair/util/ARFill.java | 4 +-
.../iotdb/library/drepair/util/LikelihoodFill.java | 2 +-
.../iotdb/library/drepair/util/LinearFill.java | 2 +-
.../iotdb/library/drepair/util/LsGreedy.java | 2 +-
.../apache/iotdb/library/drepair/util/MAFill.java | 2 +-
.../iotdb/library/drepair/util/MeanFill.java | 2 +-
.../iotdb/library/drepair/util/PreviousFill.java | 2 +-
.../apache/iotdb/library/drepair/util/Screen.java | 2 +-
.../iotdb/library/drepair/util/ScreenFill.java | 4 +-
.../library/drepair/util/TimestampRepair.java | 4 +-
.../iotdb/library/drepair/util/ValueFill.java | 6 +-
.../iotdb/library/drepair/util/ValueRepair.java | 4 +-
.../apache/iotdb/library/frequency/UDTFConv.java | 14 +-
.../apache/iotdb/library/frequency/UDTFDWT.java | 14 +-
.../apache/iotdb/library/frequency/UDTFDeconv.java | 14 +-
.../apache/iotdb/library/frequency/UDTFFFT.java | 14 +-
.../iotdb/library/frequency/UDTFHighPass.java | 14 +-
.../apache/iotdb/library/frequency/UDTFIDWT.java | 14 +-
.../apache/iotdb/library/frequency/UDTFIFFT.java | 14 +-
.../iotdb/library/frequency/UDTFLowPass.java | 14 +-
.../iotdb/library/frequency/util/FFTUtil.java | 2 +-
.../library/series/UDTFConsecutiveSequences.java | 14 +-
.../library/series/UDTFConsecutiveWindows.java | 14 +-
.../iotdb/library/series/util/ConsecutiveUtil.java | 4 +-
.../iotdb/library/string/UDTFRegexMatch.java | 14 +-
.../iotdb/library/string/UDTFRegexReplace.java | 14 +-
.../iotdb/library/string/UDTFRegexSplit.java | 14 +-
.../iotdb/library/string/UDTFStrReplace.java | 14 +-
.../iotdb/library/util/LinearRegression.java | 2 +-
.../java/org/apache/iotdb/library/util/Util.java | 4 +-
.../org/apache/iotdb/commons/path/PartialPath.java | 7 +
.../org/apache/iotdb/commons}/udf/api/UDF.java | 8 +-
.../org/apache/iotdb/commons}/udf/api/UDTF.java | 20 +-
.../apache/iotdb/commons}/udf/api/access/Row.java | 2 +-
.../iotdb/commons}/udf/api/access/RowIterator.java | 2 +-
.../iotdb/commons}/udf/api/access/RowWindow.java | 6 +-
.../commons}/udf/api/collector/PointCollector.java | 21 +-
.../api/customizer/config/UDFConfigurations.java | 7 +-
.../api/customizer/config/UDTFConfigurations.java | 25 ++-
.../parameter/UDFParameterValidator.java | 14 +-
.../api/customizer/parameter/UDFParameters.java | 57 ++---
.../api/customizer/strategy/AccessStrategy.java | 13 +-
.../strategy/RowByRowAccessStrategy.java | 12 +-
.../strategy/SlidingSizeWindowAccessStrategy.java | 19 +-
.../strategy/SlidingTimeWindowAccessStrategy.java | 101 +++------
.../UDFAttributeNotProvidedException.java | 2 +-
.../commons}/udf/api/exception/UDFException.java | 2 +-
.../UDFInputSeriesDataTypeNotValidException.java | 2 +-
.../UDFInputSeriesIndexNotValidException.java | 2 +-
.../UDFInputSeriesNumberNotValidException.java | 2 +-
.../UDFOutputSeriesDataTypeNotValidException.java | 2 +-
.../exception/UDFParameterNotValidException.java | 2 +-
.../commons}/udf/builtin/BuiltinFunction.java | 2 +-
.../apache/iotdb/commons}/udf/builtin/UDTFAbs.java | 14 +-
.../iotdb/commons}/udf/builtin/UDTFAcos.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFAsin.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFAtan.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFBottomK.java | 27 ++-
.../iotdb/commons}/udf/builtin/UDTFCast.java | 42 ++--
.../iotdb/commons}/udf/builtin/UDTFCeil.java | 2 +-
.../commons}/udf/builtin/UDTFCommonDerivative.java | 8 +-
.../udf/builtin/UDTFCommonValueDifference.java | 8 +-
.../iotdb/commons}/udf/builtin/UDTFConst.java | 18 +-
.../iotdb/commons}/udf/builtin/UDTFConstE.java | 14 +-
.../iotdb/commons}/udf/builtin/UDTFConstPi.java | 14 +-
.../iotdb/commons}/udf/builtin/UDTFContains.java | 18 +-
.../udf/builtin/UDTFContinuouslySatisfy.java | 20 +-
.../apache/iotdb/commons}/udf/builtin/UDTFCos.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFCosh.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFDegrees.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFDerivative.java | 14 +-
.../udf/builtin/UDTFEqualSizeBucketAggSample.java | 20 +-
.../udf/builtin/UDTFEqualSizeBucketM4Sample.java | 20 +-
.../builtin/UDTFEqualSizeBucketOutlierSample.java | 22 +-
.../builtin/UDTFEqualSizeBucketRandomSample.java | 16 +-
.../udf/builtin/UDTFEqualSizeBucketSample.java | 8 +-
.../apache/iotdb/commons}/udf/builtin/UDTFExp.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFFloor.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFInRange.java | 20 +-
.../iotdb/commons}/udf/builtin/UDTFJexl.java | 49 ++---
.../apache/iotdb/commons}/udf/builtin/UDTFLog.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFLog10.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFMatches.java | 20 +-
.../iotdb/commons}/udf/builtin/UDTFMath.java | 20 +-
.../udf/builtin/UDTFNonNegativeDerivative.java | 8 +-
.../builtin/UDTFNonNegativeValueDifference.java | 8 +-
.../commons}/udf/builtin/UDTFNonZeroCount.java | 2 +-
.../commons}/udf/builtin/UDTFNonZeroDuration.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFOnOff.java | 18 +-
.../iotdb/commons}/udf/builtin/UDTFRadians.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFRound.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFSelectK.java | 23 +-
.../iotdb/commons}/udf/builtin/UDTFSign.java | 2 +-
.../apache/iotdb/commons}/udf/builtin/UDTFSin.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFSinh.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFSqrt.java | 2 +-
.../apache/iotdb/commons}/udf/builtin/UDTFTan.java | 2 +-
.../iotdb/commons}/udf/builtin/UDTFTanh.java | 2 +-
.../commons}/udf/builtin/UDTFTimeDifference.java | 16 +-
.../iotdb/commons}/udf/builtin/UDTFTopK.java | 4 +-
.../commons}/udf/builtin/UDTFValueDifference.java | 14 +-
.../iotdb/commons}/udf/builtin/UDTFValueTrend.java | 14 +-
.../iotdb/commons}/udf/builtin/UDTFZeroCount.java | 2 +-
.../commons}/udf/builtin/UDTFZeroDuration.java | 2 +-
.../selectinto/InsertTabletPlansIterator.java | 2 +-
.../db/mpp/common/schematree/PathPatternTree.java | 12 +-
.../execution/operator/process/FilterOperator.java | 4 +-
.../operator/process/TransformOperator.java | 16 +-
.../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 2 +-
.../apache/iotdb/db/mpp/plan/analyze/Analyzer.java | 46 +++-
.../db/mpp/plan/analyze/ConcatPathRewriter.java | 2 +-
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 38 ++--
.../iotdb/db/mpp/plan/analyze/ExpressionUtils.java | 50 ++---
.../mpp/plan/analyze/GroupByLevelController.java | 6 +-
.../{query => mpp/plan}/expression/Expression.java | 60 +++---
.../plan}/expression/ExpressionType.java | 2 +-
.../plan}/expression/ResultColumn.java | 2 +-
.../expression/binary/AdditionExpression.java | 12 +-
.../binary/ArithmeticBinaryExpression.java | 4 +-
.../plan}/expression/binary/BinaryExpression.java | 28 +--
.../expression/binary/CompareBinaryExpression.java | 4 +-
.../expression/binary/DivisionExpression.java | 12 +-
.../plan}/expression/binary/EqualToExpression.java | 12 +-
.../expression/binary/GreaterEqualExpression.java | 12 +-
.../expression/binary/GreaterThanExpression.java | 12 +-
.../expression/binary/LessEqualExpression.java | 12 +-
.../expression/binary/LessThanExpression.java | 12 +-
.../expression/binary/LogicAndExpression.java | 12 +-
.../expression/binary/LogicBinaryExpression.java | 4 +-
.../plan}/expression/binary/LogicOrExpression.java | 12 +-
.../plan}/expression/binary/ModuloExpression.java | 12 +-
.../binary/MultiplicationExpression.java | 12 +-
.../expression/binary/NonEqualExpression.java | 12 +-
.../expression/binary/SubtractionExpression.java | 12 +-
.../plan}/expression/leaf/ConstantOperand.java | 20 +-
.../plan}/expression/leaf/LeafOperand.java | 6 +-
.../plan}/expression/leaf/TimeSeriesOperand.java | 24 +--
.../plan}/expression/leaf/TimestampOperand.java | 24 +--
.../plan}/expression/multi/FunctionExpression.java | 76 ++++---
.../plan}/expression/unary/InExpression.java | 12 +-
.../plan}/expression/unary/LikeExpression.java | 14 +-
.../plan}/expression/unary/LogicNotExpression.java | 20 +-
.../plan}/expression/unary/NegationExpression.java | 20 +-
.../plan}/expression/unary/RegularExpression.java | 12 +-
.../plan}/expression/unary/UnaryExpression.java | 26 +--
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 89 +++++---
.../db/mpp/plan/parser/StatementGenerator.java | 12 +-
.../db/mpp/plan/planner/LocalExecutionPlanner.java | 2 +-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 12 +-
.../iotdb/db/mpp/plan/planner/LogicalPlanner.java | 2 +-
.../mpp/plan/planner/plan/node/PlanNodeType.java | 14 +-
.../node/metedata/write/DeleteTimeSeriesNode.java | 100 +++++++++
.../metedata/write/InvalidateSchemaCacheNode.java | 127 +++++++++++
.../plan/planner/plan/node/process/FilterNode.java | 2 +-
.../planner/plan/node/process/FilterNullNode.java | 2 +-
.../planner/plan/node/process/TransformNode.java | 2 +-
.../planner/plan/node/write/DeleteDataNode.java | 100 +++++++++
.../plan/parameter/AggregationDescriptor.java | 2 +-
.../plan/parameter/FilterNullParameter.java | 2 +-
.../db/mpp/plan/statement/StatementVisitor.java | 5 +
.../statement/component/FilterNullComponent.java | 2 +-
.../mpp/plan/statement/component/ResultColumn.java | 2 +-
.../plan/statement/component/SelectComponent.java | 2 +-
.../plan/statement/component/WhereCondition.java | 2 +-
.../db/mpp/plan/statement/crud/QueryStatement.java | 4 +-
.../metadata/DeleteTimeSeriesStatement.java} | 27 +--
.../transformation/api}/LayerPointReader.java | 2 +-
.../transformation/api}/LayerRowReader.java | 4 +-
.../transformation/api}/LayerRowWindowReader.java | 4 +-
...ializableRowRecordListBackedMultiColumnRow.java | 4 +-
...izableRowRecordListBackedMultiColumnWindow.java | 10 +-
...wRecordListBackedMultiColumnWindowIterator.java | 8 +-
...ticSerializableTVListBackedSingleColumnRow.java | 6 +-
...SerializableTVListBackedSingleColumnWindow.java | 10 +-
...ableTVListBackedSingleColumnWindowIterator.java | 8 +-
.../LayerPointReaderBackedSingleColumnRow.java | 6 +-
.../transformation/dag/builder}/DAGBuilder.java | 15 +-
.../dag/builder}/EvaluationDAGBuilder.java | 15 +-
.../dag/input/ConstantInputReader.java} | 9 +-
.../dag/input/QueryDataSetInputLayer.java} | 21 +-
.../dag/input}/TsBlockInputDataSet.java | 2 +-
.../intermediate}/ConstantIntermediateLayer.java | 18 +-
.../dag/intermediate}/IntermediateLayer.java | 16 +-
.../MultiInputColumnIntermediateLayer.java | 26 +--
...InputColumnMultiReferenceIntermediateLayer.java | 31 +--
...nputColumnSingleReferenceIntermediateLayer.java | 28 +--
.../dag/memory}/LayerMemoryAssigner.java | 6 +-
.../transformation/dag/memory}/SafetyLine.java | 4 +-
.../dag}/transformer/Transformer.java | 4 +-
.../binary/ArithmeticAdditionTransformer.java | 4 +-
.../binary/ArithmeticBinaryTransformer.java | 4 +-
.../binary/ArithmeticDivisionTransformer.java | 4 +-
.../binary/ArithmeticModuloTransformer.java | 4 +-
.../ArithmeticMultiplicationTransformer.java | 4 +-
.../binary/ArithmeticSubtractionTransformer.java | 4 +-
.../dag}/transformer/binary/BinaryTransformer.java | 6 +-
.../binary/CompareBinaryTransformer.java | 4 +-
.../binary/CompareEqualToTransformer.java | 4 +-
.../binary/CompareGreaterEqualTransformer.java | 4 +-
.../binary/CompareGreaterThanTransformer.java | 4 +-
.../binary/CompareLessEqualTransformer.java | 4 +-
.../binary/CompareLessThanTransformer.java | 4 +-
.../binary/CompareNonEqualTransformer.java | 4 +-
.../transformer/binary/LogicAndTransformer.java | 5 +-
.../transformer/binary/LogicBinaryTransformer.java | 4 +-
.../transformer/binary/LogicOrTransformer.java | 4 +-
.../transformer/multi/UDFQueryRowTransformer.java | 6 +-
.../multi/UDFQueryRowWindowTransformer.java | 6 +-
.../transformer/multi/UDFQueryTransformer.java | 8 +-
.../unary/ArithmeticNegationTransformer.java | 4 +-
.../dag}/transformer/unary/InTransformer.java | 4 +-
.../transformer/unary/LogicNotTransformer.java | 4 +-
.../dag}/transformer/unary/RegularTransformer.java | 4 +-
.../transformer/unary/TransparentTransformer.java | 4 +-
.../dag}/transformer/unary/UnaryTransformer.java | 6 +-
.../transformation/dag/udf}/UDTFContext.java | 8 +-
.../transformation/dag/udf}/UDTFExecutor.java | 82 +++----
.../transformation/dag/udf}/UDTFTypeInferrer.java | 33 +--
.../transformation/dag/util}/InputRowUtils.java | 2 +-
.../transformation/dag/util}/LayerCacheUtils.java | 8 +-
.../transformation}/datastructure/Cache.java | 2 +-
.../datastructure/SerializableList.java | 4 +-
.../row/ElasticSerializableRowRecordList.java | 19 +-
.../row/SerializableRowRecordList.java | 4 +-
.../tv/ElasticSerializableBinaryTVList.java | 28 ++-
.../tv/ElasticSerializableTVList.java | 27 ++-
.../datastructure/tv/SerializableBinaryTVList.java | 2 +-
.../tv/SerializableBooleanTVList.java | 2 +-
.../datastructure/tv/SerializableDoubleTVList.java | 2 +-
.../datastructure/tv/SerializableFloatTVList.java | 2 +-
.../datastructure/tv/SerializableIntTVList.java | 2 +-
.../datastructure/tv/SerializableLongTVList.java | 2 +-
.../datastructure/tv/SerializableTVList.java | 10 +-
.../protocol/influxdb/function/InfluxFunction.java | 4 +-
.../influxdb/function/InfluxFunctionFactory.java | 2 +-
.../function/aggregator/InfluxAggregator.java | 2 +-
.../function/aggregator/InfluxCountFunction.java | 2 +-
.../function/aggregator/InfluxMeanFunction.java | 2 +-
.../function/aggregator/InfluxMedianFunction.java | 2 +-
.../function/aggregator/InfluxModeFunction.java | 2 +-
.../function/aggregator/InfluxSpreadFunction.java | 2 +-
.../function/aggregator/InfluxStddevFunction.java | 2 +-
.../function/aggregator/InfluxSumFunction.java | 2 +-
.../function/selector/InfluxFirstFunction.java | 2 +-
.../function/selector/InfluxLastFunction.java | 2 +-
.../function/selector/InfluxMaxFunction.java | 2 +-
.../function/selector/InfluxMinFunction.java | 2 +-
.../influxdb/function/selector/InfluxSelector.java | 2 +-
.../db/protocol/influxdb/handler/QueryHandler.java | 8 +-
.../influxdb/operator/InfluxSelectComponent.java | 8 +-
.../protocol/influxdb/sql/InfluxDBSqlVisitor.java | 20 +-
.../protocol/rest/handler/QueryDataSetHandler.java | 2 +-
.../iotdb/db/qp/constant/FilterConstant.java | 2 +-
.../qp/logical/crud/AggregationQueryOperator.java | 8 +-
.../db/qp/logical/crud/LastQueryOperator.java | 6 +-
.../iotdb/db/qp/logical/crud/QueryOperator.java | 8 +-
.../iotdb/db/qp/logical/crud/SelectComponent.java | 8 +-
.../db/qp/logical/crud/SpecialClauseComponent.java | 2 +-
.../db/qp/logical/crud/UDAFQueryOperator.java | 8 +-
.../iotdb/db/qp/physical/crud/AggregationPlan.java | 2 +-
.../db/qp/physical/crud/AlignByDevicePlan.java | 2 +-
.../iotdb/db/qp/physical/crud/LastQueryPlan.java | 2 +-
.../iotdb/db/qp/physical/crud/QueryPlan.java | 2 +-
.../db/qp/physical/crud/RawDataQueryPlan.java | 2 +-
.../apache/iotdb/db/qp/physical/crud/UDAFPlan.java | 2 +-
.../apache/iotdb/db/qp/physical/crud/UDFPlan.java | 2 +-
.../apache/iotdb/db/qp/physical/crud/UDTFPlan.java | 6 +-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 79 ++++---
.../iotdb/db/qp/strategy/LogicalGenerator.java | 4 +-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 6 +-
.../apache/iotdb/db/qp/utils/DatetimeUtils.java | 12 +-
.../iotdb/db/qp/utils/GroupByLevelController.java | 6 +-
.../apache/iotdb/db/qp/utils/WildcardsRemover.java | 4 +-
.../db/query/control/QueryResourceManager.java | 4 +-
.../db/query/dataset/UDTFAlignByTimeDataSet.java | 6 +-
.../apache/iotdb/db/query/dataset/UDTFDataSet.java | 21 +-
.../db/query/dataset/UDTFNonAlignDataSet.java | 4 +-
.../query/dataset/groupby/GroupByFillDataSet.java | 2 +-
.../query/udf/service/UDFClassLoaderManager.java | 3 +-
.../udf/service/UDFRegistrationInformation.java | 2 +-
.../query/udf/service/UDFRegistrationService.java | 14 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 1 -
.../java/org/apache/iotdb/db/service/IoTDB.java | 1 -
.../service/TemporaryQueryDataFileService.java | 4 +-
.../thrift/impl/DataNodeTSIServiceImpl.java | 63 ++++++
.../db/service/thrift/impl/TSServiceImpl.java | 82 ++++++-
.../db/sync/sender/service/TransportHandler.java | 33 ++-
.../db/tools/watermark/WatermarkDetector.java | 9 +-
.../db/mpp/plan/plan/QueryLogicalPlanUtil.java | 14 +-
.../write/DeleteTimeSeriesNodeSerdeTest.java | 61 ++++++
.../node/process/AggregationNodeSerdeTest.java | 2 +-
.../plan/node/process/FilterNodeSerdeTest.java | 8 +-
.../plan/node/process/FilterNullNodeSerdeTest.java | 2 +-
.../node/process/GroupByLevelNodeSerdeTest.java | 2 +-
.../source/SeriesAggregationScanNodeSerdeTest.java | 2 +-
.../plan/node/write/DeleteDataNodeSerdeTest.java | 61 ++++++
.../write/InvalidateSchemaCacheNodeSerdeTest.java | 76 +++++++
.../influxdb/sql/InfluxDBLogicalGeneratorTest.java | 4 +-
.../ElasticSerializableRowRecordListTest.java | 3 +-
.../ElasticSerializableTVListTest.java | 17 +-
.../iotdb/db/query/udf/datastructure/LRUCache.java | 2 +
.../SerializableBinaryTVListTest.java | 4 +-
.../SerializableBooleanTVListTest.java | 4 +-
.../SerializableDoubleTVListTest.java | 4 +-
.../datastructure/SerializableFloatTVListTest.java | 4 +-
.../datastructure/SerializableIntTVListTest.java | 4 +-
.../udf/datastructure/SerializableListTest.java | 2 +-
.../datastructure/SerializableLongTVListTest.java | 4 +-
.../SerializableRowRecordListTest.java | 2 +-
.../apache/iotdb/db/query/udf/example/Adder.java | 14 +-
.../.vuepress/theme/global-components/IoTDB.vue | 72 ++++---
379 files changed, 2906 insertions(+), 1937 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
index 7106cacc91,74236f6a1c..a64d8de84e
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
@@@ -55,9 -78,9 +78,10 @@@ import org.apache.iotdb.db.mpp.plan.sta
import org.apache.iotdb.db.mpp.plan.statement.metadata.CountStorageGroupStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CountTimeSeriesStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateAlignedTimeSeriesStatement;
+import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateFunctionStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateTimeSeriesStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.DeleteStorageGroupStatement;
+ import org.apache.iotdb.db.mpp.plan.statement.metadata.DeleteTimeSeriesStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.SetStorageGroupStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.SetTTLStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowChildNodesStatement;
@@@ -76,34 -99,9 +100,11 @@@ import org.apache.iotdb.db.qp.sql.IoTDB
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CountNodesContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CountStorageGroupContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CountTimeseriesContext;
+import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CreateFunctionContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ExpressionContext;
+import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.UriContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParserBaseVisitor;
import org.apache.iotdb.db.qp.utils.DatetimeUtils;
- import org.apache.iotdb.db.query.expression.Expression;
- import org.apache.iotdb.db.query.expression.binary.AdditionExpression;
- import org.apache.iotdb.db.query.expression.binary.DivisionExpression;
- import org.apache.iotdb.db.query.expression.binary.EqualToExpression;
- import org.apache.iotdb.db.query.expression.binary.GreaterEqualExpression;
- import org.apache.iotdb.db.query.expression.binary.GreaterThanExpression;
- import org.apache.iotdb.db.query.expression.binary.LessEqualExpression;
- import org.apache.iotdb.db.query.expression.binary.LessThanExpression;
- import org.apache.iotdb.db.query.expression.binary.LogicAndExpression;
- import org.apache.iotdb.db.query.expression.binary.LogicOrExpression;
- import org.apache.iotdb.db.query.expression.binary.ModuloExpression;
- import org.apache.iotdb.db.query.expression.binary.MultiplicationExpression;
- import org.apache.iotdb.db.query.expression.binary.NonEqualExpression;
- import org.apache.iotdb.db.query.expression.binary.SubtractionExpression;
- import org.apache.iotdb.db.query.expression.leaf.ConstantOperand;
- import org.apache.iotdb.db.query.expression.leaf.TimeSeriesOperand;
- import org.apache.iotdb.db.query.expression.leaf.TimestampOperand;
- import org.apache.iotdb.db.query.expression.multi.FunctionExpression;
- import org.apache.iotdb.db.query.expression.unary.InExpression;
- import org.apache.iotdb.db.query.expression.unary.LikeExpression;
- import org.apache.iotdb.db.query.expression.unary.LogicNotExpression;
- import org.apache.iotdb.db.query.expression.unary.NegationExpression;
- import org.apache.iotdb.db.query.expression.unary.RegularExpression;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
[iotdb] 03/18: ASTVisitor
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 97c4924480dcacc2367d3fa3599eea179ef2f97e
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Thu May 19 10:40:00 2022 +0800
ASTVisitor
---
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 27 ++++++++++++++++++++++
.../metadata/CreateFunctionStatement.java | 20 ++++------------
2 files changed, 31 insertions(+), 16 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
index 8d826f98f2..86481573ec 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
@@ -54,6 +54,7 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.CountNodesStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CountStorageGroupStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CountTimeSeriesStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateAlignedTimeSeriesStatement;
+import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateFunctionStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateTimeSeriesStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.DeleteStorageGroupStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.SetStorageGroupStatement;
@@ -72,7 +73,9 @@ import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CountDevicesContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CountNodesContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CountStorageGroupContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CountTimeseriesContext;
+import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CreateFunctionContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ExpressionContext;
+import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.UriContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParserBaseVisitor;
import org.apache.iotdb.db.qp.utils.DatetimeUtils;
import org.apache.iotdb.db.query.expression.Expression;
@@ -108,6 +111,8 @@ import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.HashMap;
@@ -475,6 +480,28 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
return new CountStorageGroupStatement(path);
}
+ // Create Function
+ @Override
+ public Statement visitCreateFunction(CreateFunctionContext ctx) {
+ return new CreateFunctionStatement(
+ parseIdentifier(ctx.udfName.getText()),
+ parseStringLiteral(ctx.className.getText()),
+ parseUris(ctx.uri()));
+ }
+
+ private List<URI> parseUris(List<UriContext> uriContexts) {
+ List<URI> uris = new ArrayList<>();
+ for (UriContext uriContext : uriContexts) {
+ final String uriString = uriContext.getText();
+ try {
+ uris.add(new URI(parseStringLiteral(uriString)));
+ } catch (URISyntaxException e) {
+ throw new SemanticException(String.format("'%s' is not a legal URI.", uriString));
+ }
+ }
+ return uris;
+ }
+
/** Data Manipulation Language (DML) */
// Select Statement ========================================================================
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateFunctionStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateFunctionStatement.java
index ef353dd063..990c9b7fb1 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateFunctionStatement.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateFunctionStatement.java
@@ -27,33 +27,21 @@ import org.apache.iotdb.db.mpp.plan.statement.Statement;
import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor;
import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class CreateFunctionStatement extends Statement implements IConfigStatement {
- private String udfName;
- private String className;
+ private final String udfName;
+ private final String className;
private final List<URI> uris;
- public CreateFunctionStatement(int tokenIntType) {
+ public CreateFunctionStatement(String udfName, String className, List<URI> uris) {
super();
statementType = StatementType.CREATE_FUNCTION;
- uris = new ArrayList<>();
- }
-
- public void setUdfName(String udfName) {
this.udfName = udfName;
- }
-
- public void setClassName(String className) {
this.className = className;
- }
-
- public void addUri(String uri) throws URISyntaxException {
- uris.add(new URI(uri));
+ this.uris = uris;
}
public String getUdfName() {
[iotdb] 08/18: Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ab0fa0f6da265384b157fccc50378cdb579e4c6f
Merge: bb8d17a0ad 81b91176a0
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Thu May 19 15:46:12 2022 +0800
Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
.../iotdb/db/mpp/plan/scheduler/FragmentInstanceDispatcherImpl.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
[iotdb] 05/18: /confignode.thrift
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit dfb73e801a271c0a05ffd529334f2f95df18622a
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Thu May 19 11:46:57 2022 +0800
/confignode.thrift
---
.../plan/execution/config/CreateFunctionTask.java | 49 ++++++++++++++++++++++
.../src/main/thrift/confignode.thrift | 11 +++++
2 files changed, 60 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/CreateFunctionTask.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/CreateFunctionTask.java
index 62cfc5f965..663f533729 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/CreateFunctionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/CreateFunctionTask.java
@@ -22,12 +22,19 @@ package org.apache.iotdb.db.mpp.plan.execution.config;
import org.apache.iotdb.commons.client.IClientManager;
import org.apache.iotdb.commons.consensus.PartitionRegionId;
import org.apache.iotdb.db.client.ConfigNodeClient;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateFunctionStatement;
import com.google.common.util.concurrent.ListenableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class CreateFunctionTask implements IConfigTask {
+ private static final Logger LOGGER = LoggerFactory.getLogger(CreateFunctionTask.class);
+ private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
+
private final CreateFunctionStatement createFunctionStatement;
public CreateFunctionTask(CreateFunctionStatement createFunctionStatement) {
@@ -39,5 +46,47 @@ public class CreateFunctionTask implements IConfigTask {
IClientManager<PartitionRegionId, ConfigNodeClient> clientManager)
throws InterruptedException {
return null;
+ // SettableFuture<ConfigTaskResult> future = SettableFuture.create();
+ // if (CONFIG.isClusterMode()) {
+ // TDeleteStorageGroupsReq req =
+ // new TDeleteStorageGroupsReq(deleteStorageGroupStatement.getPrefixPath());
+ // try (ConfigNodeClient client =
+ // clientManager.borrowClient(ConfigNodeInfo.partitionRegionId)) {
+ // TSStatus tsStatus = client.deleteStorageGroups(req);
+ // if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) {
+ // LOGGER.error(
+ // "Failed to execute delete storage group {} in config node, status is {}.",
+ // deleteStorageGroupStatement.getPrefixPath(),
+ // tsStatus);
+ // future.setException(new StatementExecutionException(tsStatus));
+ // } else {
+ // future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS));
+ // }
+ // } catch (TException | IOException e) {
+ // LOGGER.error("Failed to connect to config node.");
+ // future.setException(e);
+ // }
+ // } else {
+ // try {
+ // List<PartialPath> deletePathList =
+ // deleteStorageGroupStatement.getPrefixPath().stream()
+ // .map(
+ // path -> {
+ // try {
+ // return new PartialPath(path);
+ // } catch (IllegalPathException e) {
+ // return null;
+ // }
+ // })
+ // .collect(Collectors.toList());
+ // LocalConfigNode.getInstance().deleteStorageGroups(deletePathList);
+ // } catch (MetadataException e) {
+ // future.setException(e);
+ // }
+ // future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS));
+ // }
+ // // If the action is executed successfully, return the Future.
+ // // If your operation is async, you can return the corresponding future directly.
+ // return future;
}
}
diff --git a/thrift-confignode/src/main/thrift/confignode.thrift b/thrift-confignode/src/main/thrift/confignode.thrift
index f102e2aa3a..af0e8fada9 100644
--- a/thrift-confignode/src/main/thrift/confignode.thrift
+++ b/thrift-confignode/src/main/thrift/confignode.thrift
@@ -174,6 +174,13 @@ struct TConfigNodeRegisterResp {
3: optional list<common.TConfigNodeLocation> configNodeList
}
+// UDF
+struct TCreateFuctionReq {
+ 1: required string udfName
+ 2: required string className
+ 3: required list<string> uris
+}
+
service ConfigIService {
/* DataNode */
@@ -229,4 +236,8 @@ service ConfigIService {
TConfigNodeRegisterResp registerConfigNode(TConfigNodeRegisterReq req)
common.TSStatus applyConfigNode(common.TConfigNodeLocation configNodeLocation)
+
+ /* UDF */
+
+ common.TSStatus createFunction(TCreateFuctionReq req)
}
\ No newline at end of file
[iotdb] 13/18: ConfigManager
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 00e6a633784dc96760b2e5627a5eeec0f1cc762d
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Fri May 20 21:48:59 2022 +0800
ConfigManager
---
.../java/org/apache/iotdb/confignode/manager/ConfigManager.java | 5 +++++
.../src/main/java/org/apache/iotdb/confignode/manager/Manager.java | 7 +++++++
.../confignode/persistence/executor/ConfigRequestExecutor.java | 6 ++++--
3 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index b6e06544ab..dadd79976f 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -598,6 +598,11 @@ public class ConfigManager implements Manager {
: status;
}
+ @Override
+ public UDFManager getUDFManager() {
+ return udfManager;
+ }
+
public ProcedureManager getProcedureManager() {
return procedureManager;
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/Manager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/Manager.java
index 2d5d8bb460..019458a285 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/Manager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/Manager.java
@@ -89,6 +89,13 @@ public interface Manager {
*/
LoadManager getLoadManager();
+ /**
+ * Get UDFManager
+ *
+ * @return UDFManager instance
+ */
+ UDFManager getUDFManager();
+
/**
* Register DataNode
*
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigRequestExecutor.java b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigRequestExecutor.java
index 4cd9127994..c522bc7226 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigRequestExecutor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigRequestExecutor.java
@@ -215,7 +215,8 @@ public class ConfigRequestExecutor {
}
AtomicBoolean result = new AtomicBoolean(true);
- getAllAttributes().parallelStream()
+ getAllAttributes()
+ .parallelStream()
.forEach(
x -> {
boolean takeSnapshotResult = true;
@@ -244,7 +245,8 @@ public class ConfigRequestExecutor {
return;
}
- getAllAttributes().parallelStream()
+ getAllAttributes()
+ .parallelStream()
.forEach(
x -> {
try {
[iotdb] 12/18: fix compile error
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9d5ce589ecb7f5d25718c5d9ae70bae341b5a9a1
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Fri May 20 19:08:49 2022 +0800
fix compile error
---
server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java b/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java
index 2973635822..379420cdaf 100644
--- a/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java
+++ b/server/src/main/java/org/apache/iotdb/db/client/ConfigNodeClient.java
@@ -36,7 +36,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp;
import org.apache.iotdb.confignode.rpc.thrift.TCountStorageGroupResp;
-import org.apache.iotdb.confignode.rpc.thrift.TCreateFuctionReq;
+import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfoResp;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterResp;
@@ -589,7 +589,7 @@ public class ConfigNodeClient implements ConfigIService.Iface, SyncThriftClient,
}
@Override
- public TSStatus createFunction(TCreateFuctionReq req) throws TException {
+ public TSStatus createFunction(TCreateFunctionReq req) throws TException {
for (int i = 0; i < RETRY_NUM; i++) {
try {
TSStatus status = client.createFunction(req);
[iotdb] 10/18: merge master and fix conflicts
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fe5652865f92e5f7a0f14b08dbbd80682e29ef8b
Merge: d7a195f819 94578876e6
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Fri May 20 18:21:38 2022 +0800
merge master and fix conflicts
LICENSE | 18 +-
.../log/manage/PartitionedSnapshotLogManager.java | 4 +-
.../server/handlers/caller/ElectionHandler.java | 4 +-
.../cluster/utils/nodetool/ClusterMonitor.java | 4 +-
confignode/pom.xml | 5 -
.../confignode/conf/ConfigNodeDescriptor.java | 1 -
.../consensus/request/ConfigRequest.java | 5 +
.../consensus/request/ConfigRequestType.java | 2 +
.../request/read/GetChildNodesPartitionReq.java | 67 +
.../request/read/GetChildPathsPartitionReq.java | 67 +
.../response/SchemaNodeManagementResp.java | 73 +
.../iotdb/confignode/manager/ConfigManager.java | 38 +-
.../apache/iotdb/confignode/manager/Manager.java | 15 +
.../iotdb/confignode/manager/PartitionManager.java | 31 +
.../confignode/persistence/ClusterSchemaInfo.java | 31 +
.../iotdb/confignode/persistence/NodeInfo.java | 43 +-
.../confignode/persistence/PartitionInfo.java | 50 +-
.../confignode/persistence/ProcedureInfo.java | 9 +-
.../executor/ConfigRequestExecutor.java | 43 +
.../iotdb/confignode/procedure/Procedure.java | 43 +-
.../confignode/procedure/ProcedureExecutor.java | 68 +-
.../procedure/StateMachineProcedure.java | 4 +-
.../procedure/conf/ProcedureNodeConfig.java | 155 --
.../conf/ProcedureNodeConfigDescriptor.java | 219 ---
.../procedure/conf/ProcedureNodeConstant.java | 32 -
.../procedure/store/ConfigProcedureStore.java | 6 +-
.../procedure/store/IProcedureStore.java | 4 +-
.../procedure/store/ProcedureFactory.java | 5 +-
.../confignode/procedure/store/ProcedureStore.java | 14 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 24 +
.../confignode/persistence/AuthorInfoTest.java | 6 -
.../confignode/procedure/NoopProcedureStore.java | 4 +-
.../confignode/procedure/TestProcedureBase.java | 7 +-
.../procedure/TestProcedureExecutor.java | 2 -
.../procedure/store/TestProcedureStore.java | 9 +-
.../thrift/ConfigNodeRPCServiceProcessorTest.java | 95 +-
.../GrafanaPlugin/Dockerfile-0.14.0-iotdb | 42 +
.../DockerCompose/GrafanaPlugin/docker-compose.yml | 48 +
.../GrafanaPlugin/iotdb/conf/iotdb-rest.properties | 58 +
docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 205 ++-
docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 206 ++-
.../controller/DatabaseConnectController.java | 3 +
library-udf/pom.xml | 22 +-
.../org/apache/iotdb/library/anomaly/UDTFIQR.java | 111 ++
.../apache/iotdb/library/anomaly/UDTFKSigma.java | 153 ++
.../org/apache/iotdb/library/anomaly/UDTFLOF.java | 215 +++
.../iotdb/library/anomaly/UDTFMissDetect.java | 74 +
.../apache/iotdb/library/anomaly/UDTFRange.java | 97 ++
.../iotdb/library/anomaly/UDTFTwoSidedFilter.java | 92 ++
.../iotdb/library/anomaly/util/MissDetector.java | 240 +++
.../library/anomaly/util/StreamMissDetector.java | 189 +++
.../iotdb/library/anomaly/util/WindowDetect.java | 96 ++
.../java/org/apache/iotdb/library/util/Util.java | 3 -
.../apache/iotdb/library/anomaly/AnomalyTests.java | 1534 ++++++++++++++++++++
.../dropwizard/DropwizardMetricManager.java | 32 -
.../main/assembly/resources/conf/iotdb-metric.yml | 5 +-
.../iotdb/metrics/DoNothingMetricService.java | 14 +-
.../org/apache/iotdb/metrics/MetricManager.java | 8 -
.../org/apache/iotdb/metrics/MetricService.java | 21 +-
.../apache/iotdb/metrics/config/MetricConfig.java | 3 +-
.../iotdb/metrics/impl/DoNothingMetricManager.java | 4 -
.../IMetricSet.java} | 17 +-
.../predefined/jvm/JvmClassLoaderMetrics.java | 52 +
.../metrics/predefined/jvm/JvmCompileMetrics.java | 50 +
.../iotdb/metrics/predefined/jvm/JvmGcMetrics.java | 292 ++++
.../metrics/predefined/jvm/JvmMemoryMetrics.java | 105 ++
.../iotdb/metrics/predefined/jvm/JvmMetrics.java | 49 +
.../metrics/predefined/jvm/JvmThreadMetrics.java | 87 ++
.../metrics/predefined/logback/LogbackMetrics.java | 180 +++
.../utils/{PredefinedMetric.java => JvmUtils.java} | 28 +-
.../iotdb/metrics/utils/PredefinedMetric.java | 5 +-
.../micrometer/MicrometerMetricManager.java | 51 -
.../apache/iotdb/commons/conf/CommonConfig.java | 25 +-
.../iotdb/commons/conf/CommonDescriptor.java | 1 -
.../apache/iotdb/commons/conf/IoTDBConstant.java | 3 +
.../iotdb/commons/partition/DataPartition.java | 28 +-
.../partition/SchemaNodeManagementPartition.java | 58 +
.../iotdb/commons/partition/SchemaPartition.java | 45 +-
.../iotdb/commons/partition/DataPartitionTest.java | 12 +-
.../commons/partition/SchemaPartitionTest.java | 12 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 19 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 27 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 1 -
.../apache/iotdb/db/engine/cache/ChunkCache.java | 4 +-
.../db/engine/cache/TimeSeriesMetadataCache.java | 4 +-
.../compaction/CompactionMetricsManager.java | 4 +-
.../apache/iotdb/db/engine/flush/FlushManager.java | 4 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 4 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 4 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 4 +-
.../engine/storagegroup/TsFileProcessorInfo.java | 4 +-
.../db/metadata/rescon/TimeseriesStatistics.java | 4 +-
.../apache/iotdb/db/metadata/utils/MetaUtils.java | 26 +-
.../iotdb/db/mpp/aggregation/Aggregator.java | 12 +-
.../db/mpp/aggregation/FirstValueAccumulator.java | 3 +
.../db/mpp/aggregation/LastValueAccumulator.java | 3 +
.../iotdb/db/mpp/common/header/DatasetHeader.java | 4 +
.../iotdb/db/mpp/common/header/HeaderConstant.java | 10 +
...egateOperator.java => AggregationOperator.java} | 8 +-
.../execution/operator/process/FillOperator.java | 50 +-
.../execution/operator/process/LimitOperator.java | 3 +
.../operator/process/LinearFillOperator.java | 191 +++
.../execution/operator/process/OffsetOperator.java | 3 +
...erator.java => RawDataAggregationOperator.java} | 10 +-
.../execution/operator/process/fill/IFill.java} | 13 +-
.../process/fill/constant/BinaryConstantFill.java | 63 +
.../process/fill/constant/BooleanConstantFill.java | 62 +
.../process/fill/constant/DoubleConstantFill.java | 62 +
.../process/fill/constant/FloatConstantFill.java | 62 +
.../process/fill/constant/IntConstantFill.java | 62 +
.../process/fill/constant/LongConstantFill.java | 62 +
.../process/fill/linear/DoubleLinearFill.java | 94 ++
.../process/fill/linear/FloatLinearFill.java | 94 ++
.../process/fill/linear/IntLinearFill.java | 94 ++
.../operator/process/fill/linear/LinearFill.java | 186 +++
.../process/fill/linear/LongLinearFill.java | 94 ++
.../process/fill/previous/BinaryPreviousFill.java | 86 ++
.../process/fill/previous/BooleanPreviousFill.java | 85 ++
.../process/fill/previous/DoublePreviousFill.java | 85 ++
.../process/fill/previous/FloatPreviousFill.java | 85 ++
.../process/fill/previous/IntPreviousFill.java | 85 ++
.../process/fill/previous/LongPreviousFill.java | 85 ++
.../schema/ChildNodesSchemaScanOperator.java | 93 ++
.../schema/ChildPathsSchemaScanOperator.java | 93 ++
.../NodeManageMemoryMergeOperator.java} | 51 +-
...a => AlignedSeriesAggregationScanOperator.java} | 184 +--
...tor.java => SeriesAggregationScanOperator.java} | 26 +-
.../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 192 ++-
.../apache/iotdb/db/mpp/plan/analyze/Analyzer.java | 636 ++++----
.../mpp/plan/analyze/ClusterPartitionFetcher.java | 44 +
.../db/mpp/plan/analyze/ConcatPathRewriter.java | 65 +-
.../db/mpp/plan/analyze/ExpressionAnalyzer.java | 402 +++--
.../mpp/plan/analyze/FakePartitionFetcherImpl.java | 8 +
.../db/mpp/plan/analyze/IPartitionFetcher.java | 5 +
.../plan/analyze/StandalonePartitionFetcher.java | 8 +
.../db/mpp/plan/execution/QueryExecution.java | 2 +-
.../mpp/plan/execution/config/ConfigExecution.java | 2 +-
.../memory/StatementMemorySourceVisitor.java | 39 +
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 25 +-
.../db/mpp/plan/parser/StatementGenerator.java | 4 +-
.../db/mpp/plan/planner/LocalExecutionPlanner.java | 269 +++-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 180 +--
.../iotdb/db/mpp/plan/planner/LogicalPlanner.java | 80 +-
.../mpp/plan/planner/plan/node/PlanNodeType.java | 14 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 17 +-
.../metedata/read/ChildNodesSchemaScanNode.java | 87 ++
.../metedata/read/ChildPathsSchemaScanNode.java | 87 ++
.../read/NodeManagementMemoryMergeNode.java} | 92 +-
.../plan/planner/plan/node/process/FillNode.java | 8 +
.../source/AlignedSeriesAggregationScanNode.java | 6 +-
.../plan/node/write/InsertMultiTabletsNode.java | 6 +
.../planner/plan/node/write/InsertRowNode.java | 6 +
.../planner/plan/node/write/InsertRowsNode.java | 6 +
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 6 +
.../planner/plan/node/write/InsertTabletNode.java | 6 +
.../planner/plan/parameter/FillDescriptor.java | 8 +
.../db/mpp/plan/statement/StatementVisitor.java | 10 +
.../mpp/plan/statement/component/ResultColumn.java | 18 +-
.../plan/statement/component/SelectComponent.java | 23 +-
.../db/mpp/plan/statement/crud/QueryStatement.java | 42 +-
.../mpp/plan/statement/literal/BooleanLiteral.java | 9 +-
.../mpp/plan/statement/literal/DoubleLiteral.java | 14 +-
.../db/mpp/plan/statement/literal/Literal.java | 25 +
.../db/mpp/plan/statement/literal/LongLiteral.java | 25 +-
.../mpp/plan/statement/literal/StringLiteral.java | 6 +
.../metadata/ShowChildNodesStatement.java} | 23 +-
.../metadata/ShowChildPathsStatement.java} | 23 +-
.../groupby/impl/LocalAlignedGroupByExecutor.java | 2 -
.../db/query/pool/RawQueryReadTaskPoolManager.java | 4 +-
.../iotdb/db/service/metrics/MetricsService.java | 150 +-
.../db/service/metrics/{ => enums}/Metric.java | 2 +-
.../db/service/metrics/{ => enums}/Operation.java | 2 +-
.../iotdb/db/service/metrics/{ => enums}/Tag.java | 2 +-
.../FileMetrics.java} | 132 +-
.../ProcessMetrics.java} | 51 +-
.../SystemMetrics.java} | 37 +-
.../db/service/thrift/ProcessorWithMetrics.java | 4 +-
.../thrift/impl/DataNodeTSIServiceImpl.java | 2 +-
.../service/thrift/impl/InternalServiceImpl.java | 59 +-
.../db/service/thrift/impl/TSServiceImpl.java | 2 +-
...ratorTest.java => AggregationOperatorTest.java} | 70 +-
... AlignedSeriesAggregationScanOperatorTest.java} | 436 +++---
.../mpp/execution/operator/FillOperatorTest.java | 353 +++++
.../execution/operator/LinearFillOperatorTest.java | 441 ++++++
...st.java => RawDataAggregationOperatorTest.java} | 50 +-
...java => SeriesAggregationScanOperatorTest.java} | 146 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeFailTest.java | 18 +
.../iotdb/db/mpp/plan/plan/LogicalPlannerTest.java | 40 +
.../db/mpp/plan/plan/QueryLogicalPlanUtil.java | 346 ++---
.../NodeManagementMemoryMergeNodeSerdeTest.java | 111 ++
thrift-commons/src/main/thrift/common.thrift | 2 +
.../src/main/thrift/confignode.thrift | 23 +
.../iotdb/tsfile/read/TsFileSequenceReader.java | 52 +-
.../org/apache/iotdb/tsfile/read/common/Path.java | 3 +-
194 files changed, 10348 insertions(+), 2814 deletions(-)
diff --cc confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequest.java
index 2f4220198a,26cd531973..2e169c71db
--- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequest.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequest.java
@@@ -183,8 -183,9 +184,12 @@@ public abstract class ConfigRequest imp
case ApplyConfigNode:
req = new ApplyConfigNodeReq();
break;
+ case CreateFunction:
+ req = new CreateFunctionReq();
++ break;
+ case GetChildPathsPartition:
+ req = new GetChildPathsPartitionReq();
+ break;
default:
throw new IOException("unknown PhysicalPlan type: " + typeNum);
}
diff --cc confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java
index d3844f22c8,78a69f0ffa..1922abef91
--- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java
@@@ -58,5 -58,6 +58,7 @@@ public enum ConfigRequestType
ListUserRoles,
ListRoleUsers,
ApplyConfigNode,
+ CreateFunction,
+ GetChildPathsPartition,
+ GetChildNodesPartition;
}
diff --cc confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigRequestExecutor.java
index 42be0cd33f,8f363bbb63..4cd9127994
--- a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigRequestExecutor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigRequestExecutor.java
@@@ -49,9 -53,9 +54,10 @@@ import org.apache.iotdb.confignode.pers
import org.apache.iotdb.confignode.persistence.NodeInfo;
import org.apache.iotdb.confignode.persistence.PartitionInfo;
import org.apache.iotdb.confignode.persistence.ProcedureInfo;
+import org.apache.iotdb.confignode.persistence.UDFInfo;
import org.apache.iotdb.consensus.common.DataSet;
import org.apache.iotdb.rpc.TSStatusCode;
+ import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.thrift.TException;
import org.slf4j.Logger;
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
index 86481573ec,dab981d3ea..5b6e56f242
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
@@@ -480,26 -478,24 +483,45 @@@ public class ASTVisitor extends IoTDBSq
return new CountStorageGroupStatement(path);
}
+ // Create Function
+ @Override
+ public Statement visitCreateFunction(CreateFunctionContext ctx) {
+ return new CreateFunctionStatement(
+ parseIdentifier(ctx.udfName.getText()),
+ parseStringLiteral(ctx.className.getText()),
+ parseUris(ctx.uri()));
+ }
+
+ private List<URI> parseUris(List<UriContext> uriContexts) {
+ List<URI> uris = new ArrayList<>();
+ for (UriContext uriContext : uriContexts) {
+ final String uriString = uriContext.getText();
+ try {
+ uris.add(new URI(parseStringLiteral(uriString)));
+ } catch (URISyntaxException e) {
+ throw new SemanticException(String.format("'%s' is not a legal URI.", uriString));
+ }
+ }
+ return uris;
++
+ // Show Child Paths =====================================================================
+ @Override
+ public Statement visitShowChildPaths(IoTDBSqlParser.ShowChildPathsContext ctx) {
+ if (ctx.prefixPath() != null) {
+ return new ShowChildPathsStatement(parsePrefixPath(ctx.prefixPath()));
+ } else {
+ return new ShowChildPathsStatement(new PartialPath(SQLConstant.getSingleRootArray()));
+ }
+ }
+
+ // Show Child Nodes =====================================================================
+ @Override
+ public Statement visitShowChildNodes(IoTDBSqlParser.ShowChildNodesContext ctx) {
+ if (ctx.prefixPath() != null) {
+ return new ShowChildNodesStatement(parsePrefixPath(ctx.prefixPath()));
+ } else {
+ return new ShowChildNodesStatement(new PartialPath(SQLConstant.getSingleRootArray()));
+ }
}
/** Data Manipulation Language (DML) */
[iotdb] 16/18: Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9a56cd3fbb88360012557b9842c1c137a270e644
Merge: 6fe0aff57b 107dbda6a1
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Mon May 23 16:16:36 2022 +0800
Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
.gitignore | 1 +
.../resources/conf/iotdb-confignode.properties | 21 +++++
.../iotdb/confignode/conf/ConfigNodeConf.java | 12 +++
.../iotdb/confignode/conf/ConfigNodeConstant.java | 1 +
.../confignode/conf/ConfigNodeDescriptor.java | 2 +
.../IoTDBSyntaxConventionStringLiteralIT.java | 4 +-
.../iotdb/db/integration/IoTDBUDFManagementIT.java | 12 +--
.../commons/udf/api/exception/UDFException.java | 2 +-
.../api}/exception/UDFRegistrationException.java | 6 +-
.../udf/builtin/BuiltinAggregationFunction.java | 59 ++++++++++++++
...va => BuiltinTimeSeriesGeneratingFunction.java} | 4 +-
.../iotdb/commons}/udf/service/UDFClassLoader.java | 4 +-
.../udf/service/UDFClassLoaderManager.java | 29 +++----
.../iotdb/commons}/udf/service/UDFLogWriter.java | 2 +-
.../udf/service/UDFRegistrationInformation.java | 2 +-
.../udf/service/UDFRegistrationService.java | 90 +++++++++++-----------
.../operator/process/TransformOperator.java | 4 +-
.../plan/expression/multi/FunctionExpression.java | 8 +-
.../db/mpp/transformation/dag/udf/UDTFContext.java | 2 +-
.../mpp/transformation/dag/udf/UDTFExecutor.java | 2 +-
.../transformation/dag/udf/UDTFTypeInferrer.java | 2 +-
.../apache/iotdb/db/qp/constant/SQLConstant.java | 31 +-------
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 32 ++++----
.../apache/iotdb/db/query/dataset/UDTFDataSet.java | 4 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 16 +++-
.../java/org/apache/iotdb/db/service/IoTDB.java | 16 +++-
.../iotdb/db/qp/physical/PhysicalPlanTest.java | 2 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 5 +-
28 files changed, 230 insertions(+), 145 deletions(-)
[iotdb] 15/18: Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6fe0aff57bbf05312d936c5437bc8a404c1a05d5
Merge: 966a6d30d0 ec0ca1044d
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Mon May 23 15:58:26 2022 +0800
Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
.../controller/DatabaseConnectController.java | 2 +-
.../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 2 +-
.../mpp/plan/planner/plan/node/PlanNodeType.java | 14 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 15 ++
.../plan/node/process/LastQueryMergeNode.java | 110 +++++++++++++
.../plan/node/source/AlignedLastQueryScanNode.java | 171 ++++++++++++++++++++
.../source/AlignedSeriesAggregationScanNode.java | 6 +-
.../plan/node/source/AlignedSeriesScanNode.java | 13 +-
.../plan/node/source/LastQueryScanNode.java | 173 +++++++++++++++++++++
.../node/source/SeriesAggregationScanNode.java | 6 +-
.../planner/plan/node/source/SeriesScanNode.java | 13 +-
.../mpp/plan/plan/FragmentInstanceSerdeTest.java | 9 --
.../node/process/AggregationNodeSerdeTest.java | 6 +-
.../node/process/GroupByLevelNodeSerdeTest.java | 10 +-
.../plan/plan/node/process/LimitNodeSerdeTest.java | 7 +-
.../plan/node/process/OffsetNodeSerdeTest.java | 7 +-
.../plan/plan/node/process/SortNodeSerdeTest.java | 7 +-
.../plan/node/process/TimeJoinNodeSerdeTest.java | 10 +-
.../source/SeriesAggregationScanNodeSerdeTest.java | 6 +-
.../plan/node/source/SeriesScanNodeSerdeTest.java | 7 +-
20 files changed, 498 insertions(+), 96 deletions(-)
[iotdb] 01/18: g4 & statement
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b53039c021b7fbb5bbaddfe3b10328b0ecf1863f
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Thu May 19 10:20:49 2022 +0800
g4 & statement
---
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 6 +-
.../db/mpp/plan/statement/StatementVisitor.java | 6 ++
.../metadata/CreateFunctionStatement.java | 85 ++++++++++++++++++++++
3 files changed, 96 insertions(+), 1 deletion(-)
diff --git a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
index d1b1565cec..f7c7f9ce5b 100644
--- a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
+++ b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
@@ -115,7 +115,11 @@ createTimeseriesOfSchemaTemplate
// Create Function
createFunction
- : CREATE FUNCTION udfName=identifier AS className=STRING_LITERAL
+ : CREATE FUNCTION udfName=identifier AS className=STRING_LITERAL USING uri (COMMA uri)*
+ ;
+
+uri
+ : STRING_LITERAL
;
// Create Trigger
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java
index 16bbdff8dd..5dc45cfe6c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java
@@ -32,6 +32,7 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.CountLevelTimeSeriesState
import org.apache.iotdb.db.mpp.plan.statement.metadata.CountStorageGroupStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CountTimeSeriesStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateAlignedTimeSeriesStatement;
+import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateFunctionStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateMultiTimeSeriesStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.CreateTimeSeriesStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.DeleteStorageGroupStatement;
@@ -112,6 +113,11 @@ public abstract class StatementVisitor<R, C> {
return visitStatement(showTTLStatement, context);
}
+ // UDF
+ public R visitCreateFunction(CreateFunctionStatement createFunctionStatement, C context) {
+ return visitStatement(createFunctionStatement, context);
+ }
+
/** Data Manipulation Language (DML) */
// Select Statement
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateFunctionStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateFunctionStatement.java
new file mode 100644
index 0000000000..ef353dd063
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/CreateFunctionStatement.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.plan.statement.metadata;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.mpp.plan.analyze.QueryType;
+import org.apache.iotdb.db.mpp.plan.constant.StatementType;
+import org.apache.iotdb.db.mpp.plan.statement.IConfigStatement;
+import org.apache.iotdb.db.mpp.plan.statement.Statement;
+import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class CreateFunctionStatement extends Statement implements IConfigStatement {
+
+ private String udfName;
+ private String className;
+ private final List<URI> uris;
+
+ public CreateFunctionStatement(int tokenIntType) {
+ super();
+ statementType = StatementType.CREATE_FUNCTION;
+ uris = new ArrayList<>();
+ }
+
+ public void setUdfName(String udfName) {
+ this.udfName = udfName;
+ }
+
+ public void setClassName(String className) {
+ this.className = className;
+ }
+
+ public void addUri(String uri) throws URISyntaxException {
+ uris.add(new URI(uri));
+ }
+
+ public String getUdfName() {
+ return udfName;
+ }
+
+ public String getClassName() {
+ return className;
+ }
+
+ public List<URI> getUris() {
+ return uris;
+ }
+
+ @Override
+ public <R, C> R accept(StatementVisitor<R, C> visitor, C context) {
+ return visitor.visitCreateFunction(this, context);
+ }
+
+ @Override
+ public QueryType getQueryType() {
+ return QueryType.WRITE;
+ }
+
+ @Override
+ public List<? extends PartialPath> getPaths() {
+ return Collections.emptyList();
+ }
+}
[iotdb] 07/18: ConfigNodeRPCServiceProcessor -> ConfigManager
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bb8d17a0ad3e70b5f5da2f0a18f10e0c5651844e
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Thu May 19 15:46:01 2022 +0800
ConfigNodeRPCServiceProcessor -> ConfigManager
---
.../java/org/apache/iotdb/confignode/manager/ConfigManager.java | 5 +++++
.../src/main/java/org/apache/iotdb/confignode/manager/Manager.java | 2 ++
.../confignode/service/thrift/ConfigNodeRPCServiceProcessor.java | 6 ++++++
3 files changed, 13 insertions(+)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index 7324a8a1cb..87d44c5174 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -556,6 +556,11 @@ public class ConfigManager implements Manager {
return nodeManager.applyConfigNode(applyConfigNodeReq);
}
+ @Override
+ public TSStatus createFunction(String udfName, String className, List<String> uris) {
+ throw new UnsupportedOperationException();
+ }
+
public ProcedureManager getProcedureManager() {
return procedureManager;
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/Manager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/Manager.java
index dbb95ca560..628b63f0c2 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/Manager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/Manager.java
@@ -202,4 +202,6 @@ public interface Manager {
* @return status
*/
TSStatus applyConfigNode(ApplyConfigNodeReq applyConfigNodeReq);
+
+ TSStatus createFunction(String udfName, String className, List<String> uris);
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
index c9ec115ad2..14b8af9cca 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
@@ -54,6 +54,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TCheckUserPrivilegesReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp;
import org.apache.iotdb.confignode.rpc.thrift.TCountStorageGroupResp;
+import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfoResp;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRegisterResp;
@@ -363,6 +364,11 @@ public class ConfigNodeRPCServiceProcessor implements ConfigIService.Iface {
return status;
}
+ @Override
+ public TSStatus createFunction(TCreateFunctionReq req) {
+ return configManager.createFunction(req.getUdfName(), req.getClassName(), req.getUris());
+ }
+
public void handleClientExit() {}
// TODO: Interfaces for data operations
[iotdb] 09/18: ConfigManager ConfigRequestExecutor
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d7a195f81917b1201f9efe00fc70c6f2c8d2af5e
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Thu May 19 16:56:38 2022 +0800
ConfigManager ConfigRequestExecutor
---
.../consensus/request/ConfigRequest.java | 3 +
.../consensus/request/ConfigRequestType.java | 3 +-
.../CreateFunctionReq.java} | 60 ++++++-------------
.../iotdb/confignode/manager/ConfigManager.java | 13 ++++-
.../UDFManager.java} | 61 +++++++-------------
.../UDFInfo.java} | 67 ++++++++--------------
.../executor/ConfigRequestExecutor.java | 17 ++++--
7 files changed, 92 insertions(+), 132 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequest.java b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequest.java
index dcef6a91f8..2f4220198a 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequest.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequest.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.confignode.consensus.request.read.GetSchemaPartitionReq;
import org.apache.iotdb.confignode.consensus.request.read.GetStorageGroupReq;
import org.apache.iotdb.confignode.consensus.request.write.ApplyConfigNodeReq;
import org.apache.iotdb.confignode.consensus.request.write.CreateDataPartitionReq;
+import org.apache.iotdb.confignode.consensus.request.write.CreateFunctionReq;
import org.apache.iotdb.confignode.consensus.request.write.CreateRegionsReq;
import org.apache.iotdb.confignode.consensus.request.write.CreateSchemaPartitionReq;
import org.apache.iotdb.confignode.consensus.request.write.DeleteProcedureReq;
@@ -182,6 +183,8 @@ public abstract class ConfigRequest implements IConsensusRequest {
case ApplyConfigNode:
req = new ApplyConfigNodeReq();
break;
+ case CreateFunction:
+ req = new CreateFunctionReq();
default:
throw new IOException("unknown PhysicalPlan type: " + typeNum);
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java
index d0fd464f4f..d3844f22c8 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java
@@ -57,5 +57,6 @@ public enum ConfigRequestType {
ListRolePrivilege,
ListUserRoles,
ListRoleUsers,
- ApplyConfigNode
+ ApplyConfigNode,
+ CreateFunction,
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/CreateFunctionReq.java
similarity index 50%
copy from confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java
copy to confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/CreateFunctionReq.java
index d0fd464f4f..2cfe6ffe44 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/CreateFunctionReq.java
@@ -16,46 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.confignode.consensus.request;
-public enum ConfigRequestType {
- RegisterDataNode,
- GetDataNodeInfo,
- SetStorageGroup,
- SetTTL,
- SetSchemaReplicationFactor,
- SetDataReplicationFactor,
- SetTimePartitionInterval,
- DeleteStorageGroup,
- GetStorageGroup,
- CountStorageGroup,
- CreateRegions,
- DeleteRegions,
- GetSchemaPartition,
- CreateSchemaPartition,
- GetOrCreateSchemaPartition,
- GetDataPartition,
- CreateDataPartition,
- GetOrCreateDataPartition,
- UpdateProcedure,
- DeleteProcedure,
- Author,
- CreateUser,
- CreateRole,
- DropUser,
- DropRole,
- GrantRole,
- GrantUser,
- GrantRoleToUser,
- RevokeUser,
- RevokeRole,
- RevokeRoleFromUser,
- UpdateUser,
- ListUser,
- ListRole,
- ListUserPrivilege,
- ListRolePrivilege,
- ListUserRoles,
- ListRoleUsers,
- ApplyConfigNode
+package org.apache.iotdb.confignode.consensus.request.write;
+
+import org.apache.iotdb.confignode.consensus.request.ConfigRequest;
+import org.apache.iotdb.confignode.consensus.request.ConfigRequestType;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class CreateFunctionReq extends ConfigRequest {
+
+ public CreateFunctionReq() {
+ super(ConfigRequestType.CreateFunction);
+ }
+
+ @Override
+ protected void serializeImpl(ByteBuffer buffer) {}
+
+ @Override
+ protected void deserializeImpl(ByteBuffer buffer) throws IOException {}
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index 87d44c5174..4e6267cae9 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -55,6 +55,7 @@ import org.apache.iotdb.confignode.persistence.ClusterSchemaInfo;
import org.apache.iotdb.confignode.persistence.NodeInfo;
import org.apache.iotdb.confignode.persistence.PartitionInfo;
import org.apache.iotdb.confignode.persistence.ProcedureInfo;
+import org.apache.iotdb.confignode.persistence.UDFInfo;
import org.apache.iotdb.confignode.persistence.executor.ConfigRequestExecutor;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp;
@@ -99,6 +100,9 @@ public class ConfigManager implements Manager {
/** Manage procedure */
private final ProcedureManager procedureManager;
+ /** UDF */
+ private final UDFManager udfManager;
+
public ConfigManager() throws IOException {
// Build the persistence module
NodeInfo nodeInfo = new NodeInfo();
@@ -106,11 +110,12 @@ public class ConfigManager implements Manager {
PartitionInfo partitionInfo = new PartitionInfo();
AuthorInfo authorInfo = new AuthorInfo();
ProcedureInfo procedureInfo = new ProcedureInfo();
+ UDFInfo udfInfo = new UDFInfo();
// Build state machine and executor
ConfigRequestExecutor executor =
new ConfigRequestExecutor(
- nodeInfo, clusterSchemaInfo, partitionInfo, authorInfo, procedureInfo);
+ nodeInfo, clusterSchemaInfo, partitionInfo, authorInfo, procedureInfo, udfInfo);
PartitionRegionStateMachine stateMachine = new PartitionRegionStateMachine(this, executor);
// Build the manager module
@@ -119,6 +124,7 @@ public class ConfigManager implements Manager {
this.partitionManager = new PartitionManager(this, partitionInfo);
this.permissionManager = new PermissionManager(this, authorInfo);
this.procedureManager = new ProcedureManager(this, procedureInfo);
+ this.udfManager = new UDFManager(this, udfInfo);
this.loadManager = new LoadManager(this);
this.consensusManager = new ConsensusManager(stateMachine);
@@ -558,7 +564,10 @@ public class ConfigManager implements Manager {
@Override
public TSStatus createFunction(String udfName, String className, List<String> uris) {
- throw new UnsupportedOperationException();
+ TSStatus status = confirmLeader();
+ return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ ? udfManager.createFunction(udfName, className, uris)
+ : status;
}
public ProcedureManager getProcedureManager() {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java
similarity index 50%
copy from confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java
copy to confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java
index d0fd464f4f..b5fbc250aa 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/UDFManager.java
@@ -16,46 +16,25 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.confignode.consensus.request;
-public enum ConfigRequestType {
- RegisterDataNode,
- GetDataNodeInfo,
- SetStorageGroup,
- SetTTL,
- SetSchemaReplicationFactor,
- SetDataReplicationFactor,
- SetTimePartitionInterval,
- DeleteStorageGroup,
- GetStorageGroup,
- CountStorageGroup,
- CreateRegions,
- DeleteRegions,
- GetSchemaPartition,
- CreateSchemaPartition,
- GetOrCreateSchemaPartition,
- GetDataPartition,
- CreateDataPartition,
- GetOrCreateDataPartition,
- UpdateProcedure,
- DeleteProcedure,
- Author,
- CreateUser,
- CreateRole,
- DropUser,
- DropRole,
- GrantRole,
- GrantUser,
- GrantRoleToUser,
- RevokeUser,
- RevokeRole,
- RevokeRoleFromUser,
- UpdateUser,
- ListUser,
- ListRole,
- ListUserPrivilege,
- ListRolePrivilege,
- ListUserRoles,
- ListRoleUsers,
- ApplyConfigNode
+package org.apache.iotdb.confignode.manager;
+
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.confignode.persistence.UDFInfo;
+
+import java.util.List;
+
+public class UDFManager {
+
+ private final ConfigManager configManager;
+ private final UDFInfo udfInfo;
+
+ public UDFManager(ConfigManager configManager, UDFInfo udfInfo) {
+ this.configManager = configManager;
+ this.udfInfo = udfInfo;
+ }
+
+ public TSStatus createFunction(String udfName, String className, List<String> uris) {
+ return new TSStatus();
+ }
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/UDFInfo.java
similarity index 50%
copy from confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java
copy to confignode/src/main/java/org/apache/iotdb/confignode/persistence/UDFInfo.java
index d0fd464f4f..4ba470d7c9 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigRequestType.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/UDFInfo.java
@@ -16,46 +16,29 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iotdb.confignode.consensus.request;
-
-public enum ConfigRequestType {
- RegisterDataNode,
- GetDataNodeInfo,
- SetStorageGroup,
- SetTTL,
- SetSchemaReplicationFactor,
- SetDataReplicationFactor,
- SetTimePartitionInterval,
- DeleteStorageGroup,
- GetStorageGroup,
- CountStorageGroup,
- CreateRegions,
- DeleteRegions,
- GetSchemaPartition,
- CreateSchemaPartition,
- GetOrCreateSchemaPartition,
- GetDataPartition,
- CreateDataPartition,
- GetOrCreateDataPartition,
- UpdateProcedure,
- DeleteProcedure,
- Author,
- CreateUser,
- CreateRole,
- DropUser,
- DropRole,
- GrantRole,
- GrantUser,
- GrantRoleToUser,
- RevokeUser,
- RevokeRole,
- RevokeRoleFromUser,
- UpdateUser,
- ListUser,
- ListRole,
- ListUserPrivilege,
- ListRolePrivilege,
- ListUserRoles,
- ListRoleUsers,
- ApplyConfigNode
+
+package org.apache.iotdb.confignode.persistence;
+
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.snapshot.SnapshotProcessor;
+import org.apache.iotdb.confignode.consensus.request.write.CreateFunctionReq;
+
+import org.apache.thrift.TException;
+
+import java.io.File;
+import java.io.IOException;
+
+public class UDFInfo implements SnapshotProcessor {
+
+ public TSStatus createFunction(CreateFunctionReq req) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean processTakeSnapshot(File snapshotDir) throws TException, IOException {
+ return false;
+ }
+
+ @Override
+ public void processLoadSnapshot(File snapshotDir) throws TException, IOException {}
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigRequestExecutor.java b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigRequestExecutor.java
index 4b7bc5d7ce..42be0cd33f 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigRequestExecutor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigRequestExecutor.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.confignode.consensus.request.read.GetSchemaPartitionReq;
import org.apache.iotdb.confignode.consensus.request.read.GetStorageGroupReq;
import org.apache.iotdb.confignode.consensus.request.write.ApplyConfigNodeReq;
import org.apache.iotdb.confignode.consensus.request.write.CreateDataPartitionReq;
+import org.apache.iotdb.confignode.consensus.request.write.CreateFunctionReq;
import org.apache.iotdb.confignode.consensus.request.write.CreateRegionsReq;
import org.apache.iotdb.confignode.consensus.request.write.CreateSchemaPartitionReq;
import org.apache.iotdb.confignode.consensus.request.write.DeleteProcedureReq;
@@ -48,6 +49,7 @@ import org.apache.iotdb.confignode.persistence.ClusterSchemaInfo;
import org.apache.iotdb.confignode.persistence.NodeInfo;
import org.apache.iotdb.confignode.persistence.PartitionInfo;
import org.apache.iotdb.confignode.persistence.ProcedureInfo;
+import org.apache.iotdb.confignode.persistence.UDFInfo;
import org.apache.iotdb.consensus.common.DataSet;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -75,17 +77,21 @@ public class ConfigRequestExecutor {
private final ProcedureInfo procedureInfo;
+ private final UDFInfo udfInfo;
+
public ConfigRequestExecutor(
NodeInfo nodeInfo,
ClusterSchemaInfo clusterSchemaInfo,
PartitionInfo partitionInfo,
AuthorInfo authorInfo,
- ProcedureInfo procedureInfo) {
+ ProcedureInfo procedureInfo,
+ UDFInfo udfInfo) {
this.nodeInfo = nodeInfo;
this.clusterSchemaInfo = clusterSchemaInfo;
this.partitionInfo = partitionInfo;
this.authorInfo = authorInfo;
this.procedureInfo = procedureInfo;
+ this.udfInfo = udfInfo;
}
public DataSet executorQueryPlan(ConfigRequest req)
@@ -168,6 +174,8 @@ public class ConfigRequestExecutor {
return authorInfo.authorNonQuery((AuthorReq) req);
case ApplyConfigNode:
return nodeInfo.updateConfigNodeList((ApplyConfigNodeReq) req);
+ case CreateFunction:
+ return udfInfo.createFunction((CreateFunctionReq) req);
default:
throw new UnknownPhysicalPlanTypeException(req.getType());
}
@@ -197,8 +205,7 @@ public class ConfigRequestExecutor {
}
AtomicBoolean result = new AtomicBoolean(true);
- getAllAttributes()
- .parallelStream()
+ getAllAttributes().parallelStream()
.forEach(
x -> {
boolean takeSnapshotResult = true;
@@ -227,8 +234,7 @@ public class ConfigRequestExecutor {
return;
}
- getAllAttributes()
- .parallelStream()
+ getAllAttributes().parallelStream()
.forEach(
x -> {
try {
@@ -244,6 +250,7 @@ public class ConfigRequestExecutor {
allAttributes.add(clusterSchemaInfo);
allAttributes.add(partitionInfo);
allAttributes.add(nodeInfo);
+ allAttributes.add(udfInfo);
return allAttributes;
}
}
[iotdb] 02/18: Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fc57e0e8b3b8bf70c0f1adeb9de2f71f10d0737f
Merge: b53039c021 eaeb42bee4
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Thu May 19 10:21:02 2022 +0800
Merge branch 'master' of github.com:apache/iotdb into iotdb-3227
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +-
.../confignode/persistence/ClusterSchemaInfo.java | 25 +-
.../apache/iotdb/tsfile/TsFileSequenceRead.java | 2 +-
.../iotdb/db/integration/IoTDBSnapshotIT.java | 259 +++++++++++++++++++++
.../iotdb/db/integration/IoTDBTagAlterIT.java | 4 +-
.../commons/utils/ThriftConfigNodeSerDeUtils.java | 34 +++
.../schemaregion/rocksdb/RSchemaRegion.java | 20 +-
.../statemachine/DataRegionStateMachine.java | 74 +++---
.../statemachine/SchemaRegionStateMachine.java | 6 +-
.../statemachine/visitor/DataExecutionVisitor.java | 116 +++++++++
.../apache/iotdb/db/engine/StorageEngineV2.java | 13 ++
.../db/engine/compaction/CompactionScheduler.java | 4 +-
.../db/engine/compaction/CompactionUtils.java | 4 +-
.../constant/InnerSequenceCompactionSelector.java | 8 +-
.../InnerUnsequenceCompactionSelector.java | 8 +-
...ement.java => AbstractCrossSpaceEstimator.java} | 25 +-
.../rewrite/CrossSpaceCompactionResource.java | 89 +++++++
.../RewriteCrossSpaceCompactionResource.java | 206 ----------------
.../RewriteCrossSpaceCompactionSelector.java | 17 +-
.../selector/RewriteCompactionFileSelector.java | 195 ++++------------
.../cross/utils/AbstractCompactionEstimator.java | 63 +++++
.../IFileQueryMemMeasurement.java | 2 +-
.../cross/utils/InplaceCompactionEstimator.java | 173 ++++++++++++++
.../utils/ReadPointCrossCompactionEstimator.java | 172 ++++++++++++++
.../AbstractInnerSpaceEstimator.java} | 25 +-
.../sizetiered/SizeTieredCompactionSelector.java | 9 +-
.../compaction/task/ICompactionSelector.java | 20 ++
.../iotdb/db/engine/snapshot/SnapshotLoader.java | 196 ++++++++++++++++
.../iotdb/db/engine/snapshot/SnapshotTaker.java | 166 +++++++++++++
.../exception/DirectoryNotLegalException.java} | 22 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 4 +
.../engine/storagegroup/TsFileNameGenerator.java | 2 +-
.../apache/iotdb/db/metadata/MetadataConstant.java | 5 +
.../iotdb/db/metadata/logfile/MLogWriter.java | 9 +
.../iotdb/db/metadata/mtree/MTreeAboveSG.java | 55 +++--
.../traverser/collector/MNodeAboveSGCollector.java | 6 +-
.../db/metadata/schemaregion/ISchemaRegion.java | 8 +-
.../schemaregion/SchemaRegionMemoryImpl.java | 81 ++++++-
.../schemaregion/SchemaRegionSchemaFileImpl.java | 22 +-
.../apache/iotdb/db/metadata/tag/TagLogFile.java | 12 +-
.../apache/iotdb/db/metadata/tag/TagManager.java | 34 +++
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 8 +-
.../db/mpp/plan/planner/DistributionPlanner.java | 53 ++++-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 25 ++
.../plan/node/source/AlignedSeriesScanNode.java | 12 +-
.../plan/node/write/InsertMultiTabletsNode.java | 10 +
.../plan/planner/plan/node/write/InsertNode.java | 30 +++
.../planner/plan/node/write/InsertRowNode.java | 4 +-
.../planner/plan/node/write/InsertRowsNode.java | 10 +
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 10 +
.../planner/plan/node/write/InsertTabletNode.java | 3 +
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 2 +-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 8 +-
.../apache/iotdb/db/tools/TsFileSketchTool.java | 6 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 2 +-
.../java/org/apache/iotdb/db/utils/MergeUtils.java | 69 ------
.../compaction/cross/CrossSpaceCompactionTest.java | 17 +-
.../engine/compaction/cross/MergeUpgradeTest.java | 6 +-
.../cross/RewriteCompactionFileSelectorTest.java | 81 +++----
.../iotdb/db/metadata/mtree/MTreeAboveSGTest.java | 18 +-
.../db/metadata/schemaRegion/SchemaRegionTest.java | 100 ++++++++
.../plan/node/write/InsertTabletNodeSerdeTest.java | 2 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../iotdb/tsfile/read/TsFileSequenceReader.java | 15 +-
.../tsfile/v2/read/TsFileSequenceReaderForV2.java | 3 +-
.../tsfile/write/MetadataIndexConstructorTest.java | 2 +-
.../iotdb/tsfile/write/TsFileIOWriterTest.java | 2 +-
67 files changed, 1989 insertions(+), 707 deletions(-)
[iotdb] 11/18: fix conflicts
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3227
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d451ab0ef52384a33af3a361ef8ffb8169bdbb72
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Fri May 20 19:05:29 2022 +0800
fix conflicts
---
server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
index 5b6e56f242..7106cacc91 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
@@ -503,6 +503,7 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
}
}
return uris;
+ }
// Show Child Paths =====================================================================
@Override