You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2022/05/31 10:10:28 UTC
[iotdb] branch iotdb-3229 created (now 6eec8697f1)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a change to branch iotdb-3229
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 6eec8697f1 [IOTDB-3229] Apache IoTDB UDF Management in MPP Cluster: SHOW FUNCTIONS
This branch includes the following new commits:
new 6eec8697f1 [IOTDB-3229] Apache IoTDB UDF Management in MPP Cluster: SHOW FUNCTIONS
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: [IOTDB-3229] Apache IoTDB UDF Management in MPP Cluster: SHOW FUNCTIONS
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch iotdb-3229
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6eec8697f1a95647dbbce91d4401cfe6520463f8
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Tue May 31 18:03:54 2022 +0800
[IOTDB-3229] Apache IoTDB UDF Management in MPP Cluster: SHOW FUNCTIONS
---
.../iotdb/db/mpp/common/header/HeaderConstant.java | 18 +++
.../plan/execution/config/ConfigTaskVisitor.java | 7 +
.../plan/execution/config/ShowFunctionsTask.java | 153 +++++++++++++++++++++
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 8 ++
.../db/mpp/plan/statement/StatementVisitor.java | 5 +
.../statement/metadata/ShowFunctionsStatement.java | 46 +++++++
6 files changed, 237 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/HeaderConstant.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/HeaderConstant.java
index 0f2a26469e..e48e71754c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/HeaderConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/HeaderConstant.java
@@ -64,6 +64,11 @@ public class HeaderConstant {
public static final String COLUMN_HOST_ADDRESS = "HostAdress";
public static final String COLUMN_PORT = "Port";
+ // column names for show functions statement
+ public static final String COLUMN_FUNCTION_NAME = "function name";
+ public static final String COLUMN_FUNCTION_TYPE = "function type";
+ public static final String COLUMN_FUNCTION_CLASS = "class name (UDF)";
+
// dataset header for schema statement
public static final DatasetHeader showTimeSeriesHeader;
public static final DatasetHeader showDevicesHeader;
@@ -86,6 +91,9 @@ public class HeaderConstant {
// dataset header for last query
public static final DatasetHeader LAST_QUERY_HEADER;
+ // dataset header for show functions
+ public static final DatasetHeader SHOW_FUNCTIONS_HEADER;
+
static {
countStorageGroupHeader =
new DatasetHeader(
@@ -182,4 +190,14 @@ public class HeaderConstant {
new ColumnHeader(COLUMN_PORT, TSDataType.INT32)),
true);
}
+
+ static {
+ SHOW_FUNCTIONS_HEADER =
+ new DatasetHeader(
+ Arrays.asList(
+ new ColumnHeader(COLUMN_FUNCTION_NAME, TSDataType.TEXT),
+ new ColumnHeader(COLUMN_FUNCTION_TYPE, TSDataType.TEXT),
+ new ColumnHeader(COLUMN_FUNCTION_CLASS, TSDataType.TEXT)),
+ true);
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ConfigTaskVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ConfigTaskVisitor.java
index 9ef88cd17b..27311d6a7f 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ConfigTaskVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ConfigTaskVisitor.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.DropFunctionStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.SetStorageGroupStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.SetTTLStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowClusterStatement;
+import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowFunctionsStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowStorageGroupStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowTTLStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.UnSetTTLStatement;
@@ -110,5 +111,11 @@ public class ConfigTaskVisitor
return new DropFunctionTask(dropFunctionStatement);
}
+ @Override
+ public IConfigTask visitShowFunctions(
+ ShowFunctionsStatement showFunctionsStatement, TaskContext context) {
+ return new ShowFunctionsTask();
+ }
+
public static class TaskContext {}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ShowFunctionsTask.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ShowFunctionsTask.java
new file mode 100644
index 0000000000..c51b1d59c3
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ShowFunctionsTask.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.plan.execution.config;
+
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.consensus.PartitionRegionId;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction;
+import org.apache.iotdb.commons.udf.service.UDFRegistrationInformation;
+import org.apache.iotdb.commons.udf.service.UDFRegistrationService;
+import org.apache.iotdb.db.client.ConfigNodeClient;
+import org.apache.iotdb.db.mpp.common.header.HeaderConstant;
+import org.apache.iotdb.db.query.dataset.ListDataSet;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.RowRecord;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Arrays;
+
+import static org.apache.iotdb.commons.conf.IoTDBConstant.COLUMN_FUNCTION_CLASS;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.COLUMN_FUNCTION_NAME;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.COLUMN_FUNCTION_TYPE;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_BUILTIN_UDAF;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_BUILTIN_UDTF;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_EXTERNAL_UDAF;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_EXTERNAL_UDTF;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_NATIVE;
+
+public class ShowFunctionsTask implements IConfigTask {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ShowFunctionsTask.class);
+
+ @Override
+ public ListenableFuture<ConfigTaskResult> execute(
+ IClientManager<PartitionRegionId, ConfigNodeClient> clientManager)
+ throws InterruptedException {
+ final SettableFuture<ConfigTaskResult> future = SettableFuture.create();
+
+ try {
+ final TsBlockBuilder builder =
+ new TsBlockBuilder(HeaderConstant.SHOW_FUNCTIONS_HEADER.getRespDataTypes());
+
+ final QueryDataSet showDataSet = generateShowFunctionsListDataSet();
+ while (showDataSet.hasNextWithoutConstraint()) {
+ final RowRecord record = showDataSet.nextWithoutConstraint();
+
+ builder.getTimeColumnBuilder().writeLong(0);
+ for (int i = 0; i < 3; ++i) {
+ builder.getColumnBuilder(i).writeBinary(record.getFields().get(i).getBinaryV());
+ }
+ builder.declarePosition();
+ }
+
+ future.set(
+ new ConfigTaskResult(
+ TSStatusCode.SUCCESS_STATUS, builder.build(), HeaderConstant.SHOW_FUNCTIONS_HEADER));
+ } catch (Exception e) {
+ LOGGER.error("Failed to get functions.", e);
+ future.setException(e);
+ }
+
+ return future;
+ }
+
+ private QueryDataSet generateShowFunctionsListDataSet() {
+ ListDataSet listDataSet =
+ new ListDataSet(
+ Arrays.asList(
+ new PartialPath(COLUMN_FUNCTION_NAME, false),
+ new PartialPath(COLUMN_FUNCTION_TYPE, false),
+ new PartialPath(COLUMN_FUNCTION_CLASS, false)),
+ Arrays.asList(TSDataType.TEXT, TSDataType.TEXT, TSDataType.TEXT));
+
+ appendUDFs(listDataSet);
+ appendNativeFunctions(listDataSet);
+
+ listDataSet.sort(
+ (r1, r2) ->
+ String.CASE_INSENSITIVE_ORDER.compare(
+ r1.getFields().get(0).getStringValue(), r2.getFields().get(0).getStringValue()));
+ return listDataSet;
+ }
+
+ private void appendUDFs(ListDataSet listDataSet) {
+ for (UDFRegistrationInformation info :
+ UDFRegistrationService.getInstance().getRegistrationInformation()) {
+ RowRecord rowRecord = new RowRecord(0); // ignore timestamp
+ rowRecord.addField(Binary.valueOf(info.getFunctionName()), TSDataType.TEXT);
+ String functionType = "";
+ try {
+ if (info.isBuiltin()) {
+ if (info.isUDTF()) {
+ functionType = FUNCTION_TYPE_BUILTIN_UDTF;
+ } else if (info.isUDAF()) {
+ functionType = FUNCTION_TYPE_BUILTIN_UDAF;
+ }
+ } else {
+ if (info.isUDTF()) {
+ functionType = FUNCTION_TYPE_EXTERNAL_UDTF;
+ } else if (info.isUDAF()) {
+ functionType = FUNCTION_TYPE_EXTERNAL_UDAF;
+ }
+ }
+ } catch (InstantiationException
+ | InvocationTargetException
+ | NoSuchMethodException
+ | IllegalAccessException e) {
+ throw new RuntimeException(e.toString());
+ }
+ rowRecord.addField(Binary.valueOf(functionType), TSDataType.TEXT);
+ rowRecord.addField(Binary.valueOf(info.getClassName()), TSDataType.TEXT);
+ listDataSet.putRecord(rowRecord);
+ }
+ }
+
+ private void appendNativeFunctions(ListDataSet listDataSet) {
+ final Binary functionType = Binary.valueOf(FUNCTION_TYPE_NATIVE);
+ final Binary className = Binary.valueOf("");
+ for (String functionName : BuiltinAggregationFunction.getNativeFunctionNames()) {
+ RowRecord rowRecord = new RowRecord(0); // ignore timestamp
+ rowRecord.addField(Binary.valueOf(functionName.toUpperCase()), TSDataType.TEXT);
+ rowRecord.addField(functionType, TSDataType.TEXT);
+ rowRecord.addField(className, TSDataType.TEXT);
+ listDataSet.putRecord(rowRecord);
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
index ced604bfbf..a9ec1eda89 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java
@@ -92,6 +92,7 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowChildNodesStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowChildPathsStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowClusterStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowDevicesStatement;
+import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowFunctionsStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowStorageGroupStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowTTLStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowTimeSeriesStatement;
@@ -108,6 +109,7 @@ import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CountTimeseriesContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CreateFunctionContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.DropFunctionContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ExpressionContext;
+import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ShowFunctionsContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.UriContext;
import org.apache.iotdb.db.qp.sql.IoTDBSqlParserBaseVisitor;
import org.apache.iotdb.db.qp.utils.DatetimeUtils;
@@ -637,6 +639,12 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
return new DropFunctionStatement(parseIdentifier(ctx.udfName.getText()));
}
+ // Show Functions
+ @Override
+ public Statement visitShowFunctions(ShowFunctionsContext ctx) {
+ return new ShowFunctionsStatement();
+ }
+
// Show Child Paths =====================================================================
@Override
public Statement visitShowChildPaths(IoTDBSqlParser.ShowChildPathsContext ctx) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java
index adadae7760..b20124cbea 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java
@@ -49,6 +49,7 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowChildNodesStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowChildPathsStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowClusterStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowDevicesStatement;
+import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowFunctionsStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowStorageGroupStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowTTLStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowTimeSeriesStatement;
@@ -145,6 +146,10 @@ public abstract class StatementVisitor<R, C> {
return visitStatement(dropFunctionStatement, context);
}
+ public R visitShowFunctions(ShowFunctionsStatement showFunctionsStatement, C context) {
+ return visitStatement(showFunctionsStatement, context);
+ }
+
/** Data Manipulation Language (DML) */
// Select Statement
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/ShowFunctionsStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/ShowFunctionsStatement.java
new file mode 100644
index 0000000000..269fd07dbe
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/ShowFunctionsStatement.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.plan.statement.metadata;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.mpp.plan.analyze.QueryType;
+import org.apache.iotdb.db.mpp.plan.statement.IConfigStatement;
+import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor;
+
+import java.util.Collections;
+import java.util.List;
+
+public class ShowFunctionsStatement extends ShowStatement implements IConfigStatement {
+
+ @Override
+ public <R, C> R accept(StatementVisitor<R, C> visitor, C context) {
+ return visitor.visitShowFunctions(this, context);
+ }
+
+ @Override
+ public QueryType getQueryType() {
+ return QueryType.READ;
+ }
+
+ @Override
+ public List<PartialPath> getPaths() {
+ return Collections.emptyList();
+ }
+}