You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2024/07/05 09:41:17 UTC
(iotdb) branch ty/ValidateSensorBug created (now 500e26df9d7)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch ty/ValidateSensorBug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 500e26df9d7 Fix some bugs in query schema fetch
This branch includes the following new commits:
new 500e26df9d7 Fix some bugs in query schema fetch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: Fix some bugs in query schema fetch
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/ValidateSensorBug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 500e26df9d7818c653bb9dc985f69abdc12e62b7
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Fri Jul 5 17:41:04 2024 +0800
Fix some bugs in query schema fetch
---
.../plan/relational/metadata/Metadata.java | 3 +-
.../relational/metadata/TableMetadataImpl.java | 6 +-
.../metadata/fetcher/TableDeviceSchemaFetcher.java | 112 ++++++++++---------
.../fetcher/TableDeviceSchemaValidator.java | 119 ++++++++++++---------
.../fetcher/cache/TableDeviceCacheEntry.java | 25 +++--
.../planner/node/CreateTableDeviceNode.java | 44 ++++++--
.../optimizations/PushPredicateIntoTableScan.java | 3 +-
.../attribute/DeviceAttributeStore.java | 28 ++---
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 4 +-
.../read/resp/info/impl/ShowDevicesResult.java | 3 +-
.../plan/relational/analyzer/TestMatadata.java | 3 +-
11 files changed, 205 insertions(+), 145 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java
index 6eba3f8a196..62eee0d46a2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java
@@ -75,7 +75,8 @@ public interface Metadata {
List<DeviceEntry> indexScan(
QualifiedObjectName tableName,
List<Expression> expressionList,
- List<String> attributeColumns);
+ List<String> attributeColumns,
+ MPPQueryContext context);
/**
* This method is used for table column validation and should be invoked before device validation.
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
index 45f5708e8ff..369b29fded3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
@@ -302,13 +302,15 @@ public class TableMetadataImpl implements Metadata {
public List<DeviceEntry> indexScan(
QualifiedObjectName tableName,
List<Expression> expressionList,
- List<String> attributeColumns) {
+ List<String> attributeColumns,
+ MPPQueryContext context) {
return TableDeviceSchemaFetcher.getInstance()
.fetchDeviceSchemaForDataQuery(
tableName.getDatabaseName(),
tableName.getObjectName(),
expressionList,
- attributeColumns);
+ attributeColumns,
+ context);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
index 5ce6001c618..fd85de5572b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
@@ -41,6 +41,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.Ta
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser;
import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -65,6 +66,8 @@ import java.util.stream.Collectors;
public class TableDeviceSchemaFetcher {
+ private final SqlParser relationSqlParser = new SqlParser();
+
private static final Logger LOGGER = LoggerFactory.getLogger(TableDeviceSchemaFetcher.class);
private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
@@ -101,12 +104,12 @@ public class TableDeviceSchemaFetcher {
ExecutionResult executionResult =
coordinator.executeForTableModel(
statement,
- null,
+ relationSqlParser,
SessionManager.getInstance().getCurrSession(),
queryId,
SessionManager.getInstance()
.getSessionInfo(SessionManager.getInstance().getCurrSession()),
- "",
+ "Fetch Device for insert",
LocalExecutionPlanner.getInstance().metadata,
config.getQueryTimeoutThreshold());
if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
@@ -135,26 +138,11 @@ public class TableDeviceSchemaFetcher {
Column[] columns = tsBlock.get().getValueColumns();
for (int i = 0; i < tsBlock.get().getPositionCount(); i++) {
String[] nodes = new String[idLength];
- int idIndex = 0;
Map<String, String> attributeMap = new HashMap<>();
- for (int j = 0; j < columnHeaderList.size(); j++) {
- TsTableColumnSchema columnSchema =
- tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName());
- if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) {
- if (columns[j].isNull(i)) {
- nodes[idIndex] = null;
- } else {
- nodes[idIndex] = columns[j].getBinary(i).toString();
- }
- idIndex++;
- } else {
- if (columns[j].isNull(i)) {
- attributeMap.put(columnSchema.getColumnName(), null);
- } else {
- attributeMap.put(columnSchema.getColumnName(), columns[j].getBinary(i).toString());
- }
- }
- }
+
+ constructNodsArrayAndAttributeMap(
+ attributeMap, nodes, 0, columnHeaderList, columns, tableInstance, i);
+
fetchedDeviceSchema.put(new TableDeviceId(nodes), attributeMap);
}
}
@@ -171,7 +159,8 @@ public class TableDeviceSchemaFetcher {
String database,
String table,
List<Expression> expressionList,
- List<String> attributeColumns) {
+ List<String> attributeColumns,
+ MPPQueryContext queryContext) {
List<DeviceEntry> deviceEntryList = new ArrayList<>();
TsTable tableInstance = DataNodeTableCache.getInstance().getTable(database, table);
@@ -252,7 +241,9 @@ public class TableDeviceSchemaFetcher {
idPredicateForFetch,
compactedIdFuzzyPredicate,
deviceEntryList,
- idSingleMatchIndexList.size() == idPredicateList.size());
+ // only cache those exact device query
+ idSingleMatchIndexList.size() == idPredicateList.size(),
+ queryContext);
}
// TODO table metadata: implement deduplicate during schemaRegion execution
@@ -307,7 +298,8 @@ public class TableDeviceSchemaFetcher {
List<List<Expression>> idDeterminedPredicateList,
Expression idFuzzyPredicate,
List<DeviceEntry> deviceEntryList,
- boolean cacheFetchedDevice) {
+ boolean cacheFetchedDevice,
+ MPPQueryContext mppQueryContext) {
String table = tableInstance.getTableName();
@@ -317,12 +309,14 @@ public class TableDeviceSchemaFetcher {
ExecutionResult executionResult =
coordinator.executeForTableModel(
statement,
- null,
+ relationSqlParser,
SessionManager.getInstance().getCurrSession(),
queryId,
SessionManager.getInstance()
.getSessionInfo(SessionManager.getInstance().getCurrSession()),
- "",
+ String.format(
+ "fetch device for query %s : %s",
+ mppQueryContext.getQueryId(), mppQueryContext.getSql()),
LocalExecutionPlanner.getInstance().metadata,
config.getQueryTimeoutThreshold());
if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
@@ -334,8 +328,6 @@ public class TableDeviceSchemaFetcher {
List<ColumnHeader> columnHeaderList =
coordinator.getQueryExecution(queryId).getDatasetHeader().getColumnHeaders();
int idLength = DataNodeTableCache.getInstance().getTable(database, table).getIdNums();
- Map<String, String> attributeMap;
-
Throwable t = null;
try {
while (coordinator.getQueryExecution(queryId).hasNextResult()) {
@@ -353,31 +345,11 @@ public class TableDeviceSchemaFetcher {
for (int i = 0; i < tsBlock.get().getPositionCount(); i++) {
String[] nodes = new String[idLength + 1];
nodes[0] = table;
- int idIndex = 0;
- attributeMap = new HashMap<>();
- for (int j = 0; j < columnHeaderList.size(); j++) {
- TsTableColumnSchema columnSchema =
- tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName());
- if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) {
- if (columns[j].isNull(i)) {
- nodes[idIndex + 1] = null;
- } else {
- nodes[idIndex + 1] =
- columns[j].getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET);
- }
- idIndex++;
- } else {
- if (columns[j].isNull(i)) {
- attributeMap.put(columnSchema.getColumnName(), null);
- } else {
- attributeMap.put(
- columnSchema.getColumnName(),
- columns[j].getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET));
- }
- }
- }
+ Map<String, String> attributeMap = new HashMap<>();
+ constructNodsArrayAndAttributeMap(
+ attributeMap, nodes, 1, columnHeaderList, columns, tableInstance, i);
IDeviceID deviceID = IDeviceID.Factory.DEFAULT_FACTORY.create(nodes);
- // TODO table metadata: add memory control
+ // TODO table metadata: add memory control in query
deviceEntryList.add(
new DeviceEntry(
deviceID,
@@ -394,4 +366,40 @@ public class TableDeviceSchemaFetcher {
coordinator.cleanupQueryExecution(queryId, null, t);
}
}
+
+ private void constructNodsArrayAndAttributeMap(
+ Map<String, String> attributeMap,
+ String[] nodes,
+ int startIndex,
+ List<ColumnHeader> columnHeaderList,
+ Column[] columns,
+ TsTable tableInstance,
+ int rowIndex) {
+ for (int j = 0; j < columnHeaderList.size(); j++) {
+ TsTableColumnSchema columnSchema =
+ tableInstance.getColumnSchema(columnHeaderList.get(j).getColumnName());
+ // means that TsTable tableInstance which previously fetched is outdated, but it's ok that we
+ // ignore that newly added column here
+ if (columnSchema == null) {
+ continue;
+ }
+ if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) {
+ if (columns[j].isNull(rowIndex)) {
+ nodes[startIndex] = null;
+ } else {
+ nodes[startIndex] =
+ columns[j].getBinary(rowIndex).getStringValue(TSFileConfig.STRING_CHARSET);
+ }
+ startIndex++;
+ } else {
+ if (columns[j].isNull(rowIndex)) {
+ attributeMap.put(columnSchema.getColumnName(), null);
+ } else {
+ attributeMap.put(
+ columnSchema.getColumnName(),
+ columns[j].getBinary(rowIndex).getStringValue(TSFileConfig.STRING_CHARSET));
+ }
+ }
+ }
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
index dd9ccbe4765..8dd2f5885c1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher;
import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.protocol.session.SessionManager;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.plan.Coordinator;
@@ -32,16 +33,19 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.ITableDeviceSche
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceId;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDevice;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FetchDevice;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser;
import org.apache.iotdb.rpc.TSStatusCode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
public class TableDeviceSchemaValidator {
+ private final SqlParser relationSqlParser = new SqlParser();
private static final Logger LOGGER = LoggerFactory.getLogger(TableDeviceSchemaValidator.class);
@@ -91,24 +95,22 @@ public class TableDeviceSchemaValidator {
Map<String, String> attributeMap =
fetcher
.getTableDeviceCache()
- .getDeviceAttribute(database, tableName, parseArray(deviceIdList.get(i)));
+ .getDeviceAttribute(database, tableName, parseDeviceIdArray(deviceIdList.get(i)));
if (attributeMap == null) {
result.missingDeviceIndexList.add(i);
- continue;
- }
- Object[] deviceAttributeValueList = attributeValueList.get(i);
- for (int j = 0; j < attributeKeyList.size(); j++) {
- String value = attributeMap.get(attributeKeyList.get(j));
- if (deviceAttributeValueList[j] == null) {
- continue;
- }
- if (value == null) {
- result.attributeMissingInCacheDeviceIndexList.add(i);
- break;
- } else {
- if (!value.equals(String.valueOf(deviceAttributeValueList[j]))) {
- result.attributeUpdateDeviceIndexList.add(i);
- break;
+ } else {
+ Object[] deviceAttributeValueList = attributeValueList.get(i);
+ for (int j = 0, attributeSize = attributeKeyList.size(); j < attributeSize; j++) {
+ if (deviceAttributeValueList[j] != null) {
+ String key = attributeKeyList.get(j);
+ String value = attributeMap.get(key);
+ if (value == null) {
+ result.attributeMissingInCacheDeviceIndexList.add(i);
+ break;
+ } else if (!value.equals(deviceAttributeValueList[j])) {
+ result.attributeUpdateDeviceIndexList.add(i);
+ break;
+ }
}
}
}
@@ -116,12 +118,22 @@ public class TableDeviceSchemaValidator {
return result;
}
- private String[] parseArray(Object[] objects) {
+ // we need to truncate the tailing null
+ private String[] parseDeviceIdArray(Object[] objects) {
String[] strings = new String[objects.length];
+ int lastNonNullIndex = -1;
for (int i = 0; i < objects.length; i++) {
- strings[i] = objects[i] == null ? null : String.valueOf(objects[i]);
+ if (objects[i] != null) {
+ strings[i] = (String) objects[i];
+ lastNonNullIndex = i;
+ }
+ }
+ if (lastNonNullIndex == -1) {
+ throw new SemanticException("We don't support all IDColumns are null.");
}
- return strings;
+ return lastNonNullIndex == objects.length - 1
+ ? strings
+ : Arrays.copyOf(strings, lastNonNullIndex + 1);
}
private ValidateResult fetchAndValidateDeviceSchema(
@@ -155,48 +167,31 @@ public class TableDeviceSchemaValidator {
entry.getValue());
}
+ List<String> attributeKeyList = schemaValidation.getAttributeColumnNameList();
+ List<Object[]> attributeValueList = schemaValidation.getAttributeValueList();
+
ValidateResult result = new ValidateResult();
for (int index : previousValidateResult.missingDeviceIndexList) {
Object[] deviceId = schemaValidation.getDeviceIdList().get(index);
Map<String, String> attributeMap =
- fetchedDeviceSchema.get(new TableDeviceId(parseArray(deviceId)));
+ fetchedDeviceSchema.get(new TableDeviceId(parseDeviceIdArray(deviceId)));
if (attributeMap == null) {
result.missingDeviceIndexList.add(index);
} else {
- Object[] deviceAttributeValueList = schemaValidation.getAttributeValueList().get(index);
- for (int j = 0; j < schemaValidation.getAttributeColumnNameList().size(); j++) {
- String key = schemaValidation.getAttributeColumnNameList().get(j);
- String value = attributeMap.get(key);
- if (value == null) {
- if (deviceAttributeValueList[j] != null) {
- result.attributeUpdateDeviceIndexList.add(index);
- }
- } else {
- if (deviceAttributeValueList[j] == null
- || !value.equals(String.valueOf(deviceAttributeValueList[j]))) {
- result.attributeUpdateDeviceIndexList.add(index);
- }
- }
- }
+ constructAttributeUpdateDeviceIndexList(
+ attributeKeyList, attributeValueList, result, index, attributeMap);
}
}
for (int index : previousValidateResult.attributeMissingInCacheDeviceIndexList) {
Object[] deviceId = schemaValidation.getDeviceIdList().get(index);
Map<String, String> attributeMap =
- fetchedDeviceSchema.get(new TableDeviceId(parseArray(deviceId)));
+ fetchedDeviceSchema.get(new TableDeviceId(parseDeviceIdArray(deviceId)));
if (attributeMap == null) {
throw new IllegalStateException("Device shall exist but not exist.");
} else {
- for (int j = 0; j < schemaValidation.getAttributeColumnNameList().size(); j++) {
- String key = schemaValidation.getAttributeColumnNameList().get(j);
- String value = attributeMap.get(key);
- if (value == null
- || !value.equals(schemaValidation.getAttributeValueList().get(index)[j])) {
- result.attributeUpdateDeviceIndexList.add(index);
- break;
- }
- }
+ constructAttributeUpdateDeviceIndexList(
+ attributeKeyList, attributeValueList, result, index, attributeMap);
}
}
@@ -206,15 +201,35 @@ public class TableDeviceSchemaValidator {
return result;
}
+ private void constructAttributeUpdateDeviceIndexList(
+ List<String> attributeKeyList,
+ List<Object[]> attributeValueList,
+ ValidateResult result,
+ int index,
+ Map<String, String> attributeMap) {
+ Object[] deviceAttributeValueList = attributeValueList.get(index);
+ for (int j = 0, size = attributeKeyList.size(); j < size; j++) {
+ if (deviceAttributeValueList[j] != null) {
+ String key = attributeKeyList.get(j);
+ String value = attributeMap.get(key);
+
+ if (!deviceAttributeValueList[j].equals(value)) {
+ result.attributeUpdateDeviceIndexList.add(index);
+ break;
+ }
+ }
+ }
+ }
+
private void autoCreateDeviceSchema(
ITableDeviceSchemaValidation schemaValidation,
ValidateResult previousValidateResult,
MPPQueryContext context) {
- List<Object[]> deviceIdList =
- new ArrayList<>(
- previousValidateResult.missingDeviceIndexList.size()
- + previousValidateResult.attributeUpdateDeviceIndexList.size());
- List<Object[]> attributeValueList = new ArrayList<>(deviceIdList.size());
+ int size =
+ previousValidateResult.missingDeviceIndexList.size()
+ + previousValidateResult.attributeUpdateDeviceIndexList.size();
+ List<Object[]> deviceIdList = new ArrayList<>(size);
+ List<Object[]> attributeValueList = new ArrayList<>(size);
for (int index : previousValidateResult.missingDeviceIndexList) {
deviceIdList.add(schemaValidation.getDeviceIdList().get(index));
attributeValueList.add(schemaValidation.getAttributeValueList().get(index));
@@ -234,12 +249,12 @@ public class TableDeviceSchemaValidator {
ExecutionResult executionResult =
coordinator.executeForTableModel(
statement,
- null,
+ relationSqlParser,
SessionManager.getInstance().getCurrSession(),
SessionManager.getInstance().requestQueryId(),
SessionManager.getInstance()
.getSessionInfo(SessionManager.getInstance().getCurrSession()),
- "",
+ "Create device or update device attribute for insert",
LocalExecutionPlanner.getInstance().metadata,
context == null || context.getQueryType().equals(QueryType.WRITE)
? config.getQueryTimeoutThreshold()
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java
index a6d6f3f5e42..7ca4dba0ef0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceCacheEntry.java
@@ -19,22 +19,25 @@
package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache;
-import org.apache.iotdb.commons.schema.MemUsageUtil;
+import com.google.common.collect.ImmutableMap;
+import org.apache.tsfile.utils.RamUsageEstimator;
-import java.util.HashMap;
import java.util.Map;
public class TableDeviceCacheEntry {
- // should use a Map implementation that allows null for value
- private final Map<String, String> attributeMap;
+ private static final long INSTANCE_SIZE =
+ RamUsageEstimator.shallowSizeOfInstance(TableDeviceCacheEntry.class);
- public TableDeviceCacheEntry() {
- attributeMap = new HashMap<>();
- }
+ // the cached attributeMap may not be the latest, but there won't be any correctness problems
+ // because when missing getting the key-value from this attributeMap, caller will try to get or
+ // create from remote
+ // there may exist key is not null, but value is null in this map, which means that the key's
+ // corresponding value is null, doesn't mean that the key doesn't exist
+ private final ImmutableMap<String, String> attributeMap;
public TableDeviceCacheEntry(Map<String, String> attributeMap) {
- this.attributeMap = new HashMap<>(attributeMap);
+ this.attributeMap = ImmutableMap.copyOf(attributeMap);
}
public String getAttribute(String key) {
@@ -46,10 +49,6 @@ public class TableDeviceCacheEntry {
}
public int estimateSize() {
- int size = 8; // map reference
- for (Map.Entry<String, String> entry : attributeMap.entrySet()) {
- size += (int) MemUsageUtil.computeKVMemUsageInMap(entry.getKey(), entry.getValue());
- }
- return size;
+ return (int) (INSTANCE_SIZE + RamUsageEstimator.sizeOfMap(attributeMap));
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java
index 13209c2274f..f3888bc042a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/CreateTableDeviceNode.java
@@ -34,6 +34,8 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -68,11 +70,35 @@ public class CreateTableDeviceNode extends WritePlanNode {
super(id);
this.database = database;
this.tableName = tableName;
- this.deviceIdList = deviceIdList;
+ // truncate the tailing null
+ this.deviceIdList = truncateTailingNull(deviceIdList);
this.attributeNameList = attributeNameList;
this.attributeValueList = attributeValueList;
}
+ private static List<Object[]> truncateTailingNull(List<Object[]> deviceIdList) {
+ List<Object[]> res = new ArrayList<>(deviceIdList.size());
+ for (Object[] device : deviceIdList) {
+ if (device == null || device.length == 0) {
+ throw new IllegalArgumentException("DeviceID's length should be larger than 0.");
+ }
+ int lastNonNullIndex = -1;
+ for (int i = device.length - 1; i >= 0; i--) {
+ if (device[i] != null) {
+ lastNonNullIndex = i;
+ break;
+ }
+ }
+ if (lastNonNullIndex == -1) {
+ throw new IllegalArgumentException("DeviceID shouldn't be all nulls.");
+ }
+ res.add(Arrays.copyOf(device, lastNonNullIndex + 1));
+ }
+ return res;
+ }
+
+ // in this constructor, we don't need to truncate tailing nulls for deviceIdList, because this
+ // constructor can only be generated from another CreateTableDeviceNode
public CreateTableDeviceNode(
PlanNodeId id,
TRegionReplicaSet regionReplicaSet,
@@ -122,27 +148,29 @@ public class CreateTableDeviceNode extends WritePlanNode {
public List<IDeviceID> getPartitionKeyList() {
if (partitionKeyList == null) {
- List<IDeviceID> partitionKeyList = new ArrayList<>();
+ List<IDeviceID> tmpPartitionKeyList = new ArrayList<>();
for (Object[] rawId : deviceIdList) {
String[] partitionKey = new String[rawId.length + 1];
partitionKey[0] = tableName;
for (int i = 0; i < rawId.length; i++) {
- partitionKey[i + 1] = Objects.toString(rawId[i].toString());
+ partitionKey[i + 1] = (String) rawId[i];
}
- partitionKeyList.add(IDeviceID.Factory.DEFAULT_FACTORY.create(partitionKey));
+ tmpPartitionKeyList.add(IDeviceID.Factory.DEFAULT_FACTORY.create(partitionKey));
}
- this.partitionKeyList = partitionKeyList;
+ this.partitionKeyList = tmpPartitionKeyList;
}
return partitionKeyList;
}
@Override
public List<PlanNode> getChildren() {
- return new ArrayList<>();
+ return Collections.emptyList();
}
@Override
- public void addChild(PlanNode child) {}
+ public void addChild(PlanNode child) {
+ throw new UnsupportedOperationException();
+ }
@Override
public PlanNode clone() {
@@ -163,7 +191,7 @@ public class CreateTableDeviceNode extends WritePlanNode {
@Override
public List<String> getOutputColumnNames() {
- return null;
+ throw new UnsupportedOperationException();
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
index a3215c00953..98b10ab23ce 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
@@ -251,7 +251,8 @@ public class PushPredicateIntoTableScan implements TablePlanOptimizer {
.map(Symbol::getName)
.collect(Collectors.toList());
List<DeviceEntry> deviceEntries =
- metadata.indexScan(node.getQualifiedObjectName(), metadataExpressions, attributeColumns);
+ metadata.indexScan(
+ node.getQualifiedObjectName(), metadataExpressions, attributeColumns, queryContext);
node.setDeviceEntries(deviceEntries);
if (deviceEntries.isEmpty()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java
index 70125fe2831..457b48e0a6c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java
@@ -123,9 +123,11 @@ public class DeviceAttributeStore implements IDeviceAttributeStore {
Map<String, String> attributeMap = new HashMap<>();
String value;
for (int i = 0; i < nameList.size(); i++) {
- value = valueList[i] == null ? null : valueList[i].toString();
- attributeMap.put(nameList.get(i), value);
- memUsage += MemUsageUtil.computeKVMemUsageInMap(nameList.get(i), value);
+ value = valueList[i] == null ? null : (String) valueList[i];
+ if (value != null) {
+ attributeMap.put(nameList.get(i), value);
+ memUsage += MemUsageUtil.computeKVMemUsageInMap(nameList.get(i), value);
+ }
}
deviceAttributeList.add(attributeMap);
requestMemory(memUsage);
@@ -142,15 +144,17 @@ public class DeviceAttributeStore implements IDeviceAttributeStore {
String value;
for (int i = 0; i < nameList.size(); i++) {
String key = nameList.get(i);
- originMemUsage =
- attributeMap.containsKey(key)
- ? 0
- : MemUsageUtil.computeKVMemUsageInMap(key, attributeMap.get(key));
-
- value = valueList[i] == null ? null : valueList[i].toString();
- attributeMap.put(key, value);
- updatedMemUsage = MemUsageUtil.computeKVMemUsageInMap(key, value);
- memUsageDelta += updatedMemUsage - originMemUsage;
+ value = valueList[i] == null ? null : (String) valueList[i];
+ if (value != null) {
+ originMemUsage =
+ attributeMap.containsKey(key)
+ ? 0
+ : MemUsageUtil.computeKVMemUsageInMap(key, attributeMap.get(key));
+
+ attributeMap.put(key, value);
+ updatedMemUsage = MemUsageUtil.computeKVMemUsageInMap(key, value);
+ memUsageDelta += (updatedMemUsage - originMemUsage);
+ }
}
requestMemory(memUsageDelta);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
index 6663d8867a4..22fc44c4b43 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
@@ -1510,8 +1510,8 @@ public class MTreeBelowSGMemoryImpl {
cur = child;
String nodeName;
- for (int i = 0; i < devicePath.length; i++) {
- nodeName = devicePath[i] == null ? null : devicePath[i].toString();
+ for (Object o : devicePath) {
+ nodeName = o == null ? null : (String) o;
child = cur.getChild(nodeName);
if (child == null) {
child = store.addChild(cur, nodeName, nodeFactory.createInternalMNode(cur, nodeName));
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java
index b71dab7121b..dd275ca16ff 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/impl/ShowDevicesResult.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.IDeviceSchem
import java.util.Objects;
import java.util.function.Function;
+import java.util.function.UnaryOperator;
public class ShowDevicesResult extends ShowSchemaResult implements IDeviceSchemaInfo {
private Boolean isAligned;
@@ -53,7 +54,7 @@ public class ShowDevicesResult extends ShowSchemaResult implements IDeviceSchema
return templateId;
}
- public void setAttributeProvider(Function<String, String> attributeProvider) {
+ public void setAttributeProvider(UnaryOperator<String> attributeProvider) {
this.attributeProvider = attributeProvider;
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java
index bf30dc00718..0873c14c885 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java
@@ -205,7 +205,8 @@ public class TestMatadata implements Metadata {
public List<DeviceEntry> indexScan(
QualifiedObjectName tableName,
List<Expression> expressionList,
- List<String> attributeColumns) {
+ List<String> attributeColumns,
+ MPPQueryContext context) {
return Arrays.asList(
new DeviceEntry(new StringArrayDeviceID(DEVICE_4.split("\\.")), DEVICE_4_ATTRIBUTES),
new DeviceEntry(new StringArrayDeviceID(DEVICE_1.split("\\.")), DEVICE_1_ATTRIBUTES),