You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/04/28 16:06:18 UTC
[doris] branch master updated: [fix](Metadata tvf) Metadata TVF supports read the specified columns from Fe (#19110)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c74c2a4f8e [fix](Metadata tvf) Metadata TVF supports read the specified columns from Fe (#19110)
c74c2a4f8e is described below
commit c74c2a4f8eec67fef5b4c6e3e38645b1d0a66158
Author: Tiewei Fang <43...@users.noreply.github.com>
AuthorDate: Sat Apr 29 00:06:08 2023 +0800
[fix](Metadata tvf) Metadata TVF supports read the specified columns from Fe (#19110)
---
be/src/vec/exec/scan/vmeta_scanner.cpp | 7 ++++
.../catalog/external/JdbcExternalDatabase.java | 1 -
.../catalog/external/TestExternalDatabase.java | 1 -
.../tablefunction/BackendsTableValuedFunction.java | 34 +++++++++++++++
.../tablefunction/IcebergTableValuedFunction.java | 14 ++++++-
.../doris/tablefunction/MetadataGenerator.java | 49 +++++++++++++++++++---
.../ResourceGroupsTableValuedFunction.java | 11 +++++
gensrc/thrift/FrontendService.thrift | 1 +
.../table_valued_function/test_backends_tvf.groovy | 26 ++++++++++++
9 files changed, 136 insertions(+), 8 deletions(-)
diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp b/be/src/vec/exec/scan/vmeta_scanner.cpp
index 647c4c7de6..db17ea9ac4 100644
--- a/be/src/vec/exec/scan/vmeta_scanner.cpp
+++ b/be/src/vec/exec/scan/vmeta_scanner.cpp
@@ -212,6 +212,13 @@ Status VMetaScanner::_fetch_metadata(const TMetaScanRange& meta_scan_range) {
return Status::OK();
}
+ // set filter columns
+ std::vector<std::string> filter_columns;
+ for (const auto& slot : _tuple_desc->slots()) {
+ filter_columns.emplace_back(slot->col_name_lower_case());
+ }
+ request.metada_table_params.__set_columns_name(filter_columns);
+
// _state->execution_timeout() is seconds, change to milliseconds
int time_out = _state->execution_timeout() * 1000;
TNetworkAddress master_addr = ExecEnv::GetInstance()->master_info()->network_address;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalDatabase.java
index 4272d357e0..f04a389570 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalDatabase.java
@@ -115,7 +115,6 @@ public class JdbcExternalDatabase extends ExternalDatabase<JdbcExternalTable> im
initialized = true;
}
- // TODO(ftw): drew
@Override
public Set<String> getTableNamesWithLock() {
makeSureInitialized();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java
index aece45d801..fe1852241d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java
@@ -108,7 +108,6 @@ public class TestExternalDatabase extends ExternalDatabase<TestExternalTable> im
initialized = true;
}
- // TODO(ftw): drew
@Override
public Set<String> getTableNamesWithLock() {
makeSureInitialized();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/BackendsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/BackendsTableValuedFunction.java
index 864fcc8f80..fbf349c517 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/BackendsTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/BackendsTableValuedFunction.java
@@ -25,6 +25,7 @@ import org.apache.doris.thrift.TBackendsMetadataParams;
import org.apache.doris.thrift.TMetaScanRange;
import org.apache.doris.thrift.TMetadataType;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import java.util.List;
@@ -37,6 +38,39 @@ import java.util.Map;
public class BackendsTableValuedFunction extends MetadataTableValuedFunction {
public static final String NAME = "backends";
+ private static final ImmutableMap<String, Integer> COLUMN_TO_INDEX = new ImmutableMap.Builder<String, Integer>()
+ .put("backendid", 0)
+ .put("cluster", 1)
+ .put("ip", 2)
+ .put("hostname", 3)
+ .put("heartbeatport", 4)
+ .put("beport", 5)
+ .put("httpport", 6)
+ .put("brpcport", 7)
+ .put("laststarttime", 8)
+ .put("lastheartbeat", 9)
+ .put("alive", 10)
+ .put("systemdecommissioned", 11)
+ .put("clusterdecommissioned", 12)
+ .put("tabletnum", 13)
+ .put("datausedcapacity", 14)
+ .put("availcapacity", 15)
+ .put("totalcapacity", 16)
+ .put("usedpct", 17)
+ .put("maxdiskusedpct", 18)
+ .put("remoteusedcapacity", 19)
+ .put("tag", 20)
+ .put("errmsg", 21)
+ .put("version", 22)
+ .put("status", 23)
+ .put("heartbeatfailurecounter", 24)
+ .put("noderole", 25)
+ .build();
+
+ public static Integer getColumnIndexFromColumnName(String columnName) {
+ return COLUMN_TO_INDEX.get(columnName.toLowerCase());
+ }
+
public BackendsTableValuedFunction(Map<String, String> params) throws AnalysisException {
if (params.size() != 0) {
throw new AnalysisException("backends table-valued-function does not support any params");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java
index 2be9d16915..54ceb04b03 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java
@@ -31,6 +31,7 @@ import org.apache.doris.thrift.TIcebergQueryType;
import org.apache.doris.thrift.TMetaScanRange;
import org.apache.doris.thrift.TMetadataType;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
@@ -50,6 +51,18 @@ public class IcebergTableValuedFunction extends MetadataTableValuedFunction {
private static final ImmutableSet<String> PROPERTIES_SET = ImmutableSet.of(TABLE, QUERY_TYPE);
+ private static final ImmutableMap<String, Integer> COLUMN_TO_INDEX = new ImmutableMap.Builder<String, Integer>()
+ .put("committed_at", 0)
+ .put("snapshot_id", 1)
+ .put("parent_id", 2)
+ .put("operation", 3)
+ .put("manifest_list", 4)
+ .build();
+
+ public static Integer getColumnIndexFromColumnName(String columnName) {
+ return COLUMN_TO_INDEX.get(columnName.toLowerCase());
+ }
+
private TIcebergQueryType queryType;
// here tableName represents the name of a table in Iceberg.
@@ -82,7 +95,6 @@ public class IcebergTableValuedFunction extends MetadataTableValuedFunction {
this.icebergTableName.getDb() + ": " + this.icebergTableName.getTbl());
}
try {
- // TODO(ftw): check here
this.queryType = TIcebergQueryType.valueOf(queryTypeString.toUpperCase());
} catch (IllegalArgumentException e) {
throw new AnalysisException("Unsupported iceberg metadata query type: " + queryType);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index 1bfeca6612..f4dad26a99 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -33,6 +33,7 @@ import org.apache.doris.thrift.TFetchSchemaTableDataResult;
import org.apache.doris.thrift.TIcebergMetadataParams;
import org.apache.doris.thrift.TIcebergQueryType;
import org.apache.doris.thrift.TMetadataTableRequestParams;
+import org.apache.doris.thrift.TMetadataType;
import org.apache.doris.thrift.TRow;
import org.apache.doris.thrift.TStatus;
import org.apache.doris.thrift.TStatusCode;
@@ -63,17 +64,25 @@ public class MetadataGenerator {
if (!request.isSetMetadaTableParams()) {
return errorResult("Metadata table params is not set. ");
}
+ TFetchSchemaTableDataResult result;
+ TMetadataTableRequestParams params = request.getMetadaTableParams();
switch (request.getMetadaTableParams().getMetadataType()) {
case ICEBERG:
- return icebergMetadataResult(request.getMetadaTableParams());
+ result = icebergMetadataResult(params);
+ break;
case BACKENDS:
- return backendsMetadataResult(request.getMetadaTableParams());
+ result = backendsMetadataResult(params);
+ break;
case RESOURCE_GROUPS:
- return resourceGroupsMetadataResult(request.getMetadaTableParams());
- default:
+ result = resourceGroupsMetadataResult(params);
break;
+ default:
+ return errorResult("Metadata table params is not set.");
}
- return errorResult("Metadata table params is not set. ");
+ if (result.getStatus().getStatusCode() == TStatusCode.OK) {
+ filterColumns(result, params.getColumnsName(), params.getMetadataType());
+ }
+ return result;
}
@NotNull
@@ -119,6 +128,7 @@ public class MetadataGenerator {
}
trow.addToColumnValue(new TCell().setStringVal(snapshot.operation()));
trow.addToColumnValue(new TCell().setStringVal(snapshot.manifestListLocation()));
+
dataBatch.add(trow);
}
break;
@@ -232,6 +242,7 @@ public class MetadataGenerator {
// node role, show the value only when backend is alive.
trow.addToColumnValue(new TCell().setStringVal(backend.isAlive() ? backend.getNodeRoleTag().value : ""));
+
dataBatch.add(trow);
}
@@ -265,6 +276,34 @@ public class MetadataGenerator {
return result;
}
+ private static void filterColumns(TFetchSchemaTableDataResult result,
+ List<String> columnNames, TMetadataType type) {
+ List<TRow> fullColumnsRow = result.getDataBatch();
+ List<TRow> filterColumnsRows = Lists.newArrayList();
+ for (TRow row : fullColumnsRow) {
+ TRow filterRow = new TRow();
+ for (String columnName : columnNames) {
+ Integer index = 0;
+ switch (type) {
+ case ICEBERG:
+ index = IcebergTableValuedFunction.getColumnIndexFromColumnName(columnName);
+ break;
+ case BACKENDS:
+ index = BackendsTableValuedFunction.getColumnIndexFromColumnName(columnName);
+ break;
+ case RESOURCE_GROUPS:
+ index = ResourceGroupsTableValuedFunction.getColumnIndexFromColumnName(columnName);
+ break;
+ default:
+ break;
+ }
+ filterRow.addToColumnValue(row.getColumnValue().get(index));
+ }
+ filterColumnsRows.add(filterRow);
+ }
+ result.setDataBatch(filterColumnsRows);
+ }
+
private static org.apache.iceberg.Table getIcebergTable(HMSExternalCatalog catalog, String db, String tbl)
throws MetaNotFoundException {
org.apache.iceberg.hive.HiveCatalog hiveCatalog = new org.apache.iceberg.hive.HiveCatalog();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ResourceGroupsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ResourceGroupsTableValuedFunction.java
index 171bf42bf1..11a1baee49 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ResourceGroupsTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ResourceGroupsTableValuedFunction.java
@@ -24,6 +24,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.thrift.TMetaScanRange;
import org.apache.doris.thrift.TMetadataType;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import java.util.List;
@@ -35,6 +36,16 @@ import java.util.Map;
*/
public class ResourceGroupsTableValuedFunction extends MetadataTableValuedFunction {
public static final String NAME = "resource_groups";
+ private static final ImmutableMap<String, Integer> COLUMN_TO_INDEX = new ImmutableMap.Builder<String, Integer>()
+ .put("id", 0)
+ .put("name", 1)
+ .put("item", 2)
+ .put("value", 3)
+ .build();
+
+ public static Integer getColumnIndexFromColumnName(String columnName) {
+ return COLUMN_TO_INDEX.get(columnName.toLowerCase());
+ }
public ResourceGroupsTableValuedFunction(Map<String, String> params) throws AnalysisException {
if (params.size() != 0) {
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index a88edd7680..5a049ef589 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -731,6 +731,7 @@ struct TMetadataTableRequestParams {
1: optional Types.TMetadataType metadata_type
2: optional PlanNodes.TIcebergMetadataParams iceberg_metadata_params
3: optional PlanNodes.TBackendsMetadataParams backends_metadata_params
+ 4: optional list<string> columns_name
}
struct TFetchSchemaTableDataRequest {
diff --git a/regression-test/suites/correctness_p0/table_valued_function/test_backends_tvf.groovy b/regression-test/suites/correctness_p0/table_valued_function/test_backends_tvf.groovy
index 19f524ee9c..3f95bcc04b 100644
--- a/regression-test/suites/correctness_p0/table_valued_function/test_backends_tvf.groovy
+++ b/regression-test/suites/correctness_p0/table_valued_function/test_backends_tvf.groovy
@@ -20,4 +20,30 @@ suite("test_backends_tvf") {
List<List<Object>> table = sql """ select * from backends(); """
assertTrue(table.size() > 0) // row should > 0
assertTrue(table[0].size == 26) // column should be 26
+
+ // filter columns
+ table = sql """ select BackendId, HostName, Alive, TotalCapacity, Version, NodeRole from backends();"""
+ assertTrue(table.size() > 0) // row should > 0
+ assertTrue(table[0].size == 6) // column should be 26
+ assertEquals("true", table[0][2])
+
+ // case insensitive
+ table = sql """ select backendid, Hostname, alive, Totalcapacity, version, nodeRole from backends();"""
+ assertTrue(table.size() > 0) // row should > 0
+ assertTrue(table[0].size == 6) // column should be 26
+ assertEquals("true", table[0][2])
+
+ // test aliase columns
+ table = sql """ select backendid as id, Hostname as name, alive, NodeRole as r from backends();"""
+ assertTrue(table.size() > 0) // row should > 0
+ assertTrue(table[0].size == 4) // column should be 26
+ assertEquals("true", table[0][2])
+
+ // test changing position of columns
+ table = sql """ select Hostname as name, NodeRole as r, alive, ip from backends();"""
+ assertTrue(table.size() > 0) // row should > 0
+ assertTrue(table[0].size == 4) // column should be 26
+ assertEquals("true", table[0][2])
+
+
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org