You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/04/19 02:36:39 UTC
[iotdb] branch devicePathCache created (now c0216c71ec)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch devicePathCache
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at c0216c71ec Add a DevicePath cache to avoid init duplicated partial paths
This branch includes the following new commits:
new 247d4d14ca devicePathCache
new c0216c71ec Add a DevicePath cache to avoid init duplicated partial paths
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 02/02: Add a DevicePath cache to avoid init duplicated partial paths
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch devicePathCache
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c0216c71ec543929f514641a081834ba296f912d
Author: HTHou <hh...@outlook.com>
AuthorDate: Wed Apr 19 10:35:42 2023 +0800
Add a DevicePath cache to avoid init duplicated partial paths
---
.../db/metadata/cache/DataNodeDevicePathCache.java | 65 ++++++++++++++++++++++
1 file changed, 65 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/cache/DataNodeDevicePathCache.java b/server/src/main/java/org/apache/iotdb/db/metadata/cache/DataNodeDevicePathCache.java
new file mode 100644
index 0000000000..0649c27b5b
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/cache/DataNodeDevicePathCache.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.metadata.cache;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/** This class is for reducing duplicated Device PartialPath splits in write process. */
+public class DataNodeDevicePathCache {
+
+ private final Map<String, PartialPath> devicePathCache;
+
+ private DataNodeDevicePathCache() {
+ devicePathCache = new ConcurrentHashMap<>();
+ }
+
+ public static DataNodeDevicePathCache getInstance() {
+ return DataNodeDevicePathCache.DataNodeDevicePathCacheHolder.INSTANCE;
+ }
+
+ /** singleton pattern. */
+ private static class DataNodeDevicePathCacheHolder {
+ private static final DataNodeDevicePathCache INSTANCE = new DataNodeDevicePathCache();
+ }
+
+ public PartialPath get(String deviceId) throws IllegalPathException {
+ try {
+ return devicePathCache.computeIfAbsent(
+ deviceId,
+ path -> {
+ try {
+ return new PartialPath(deviceId);
+ } catch (IllegalPathException e) {
+ throw new IllegalArgumentException(e);
+ }
+ });
+ } catch (IllegalArgumentException e) {
+ throw new IllegalPathException(deviceId);
+ }
+ }
+
+ public void cleanUp() {
+ devicePathCache.clear();
+ }
+}
[iotdb] 01/02: devicePathCache
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch devicePathCache
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 247d4d14cab58495af831714b296554acd72d774
Author: HTHou <hh...@outlook.com>
AuthorDate: Wed Apr 19 10:03:31 2023 +0800
devicePathCache
---
.../org/apache/iotdb/db/audit/AuditLogger.java | 7 +++++--
.../db/mpp/plan/parser/StatementGenerator.java | 22 +++++++++++++---------
.../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 4 ++--
.../v1/handler/StatementConstructionHandler.java | 5 +++--
.../v2/handler/StatementConstructionHandler.java | 5 +++--
.../impl/DataNodeInternalRPCServiceImpl.java | 2 ++
6 files changed, 28 insertions(+), 17 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java b/server/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java
index 0355fbbfcd..147ee8637f 100644
--- a/server/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java
+++ b/server/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java
@@ -20,10 +20,10 @@ package org.apache.iotdb.db.audit;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IllegalPathException;
-import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.cache.DataNodeDevicePathCache;
import org.apache.iotdb.db.mpp.plan.Coordinator;
import org.apache.iotdb.db.mpp.plan.analyze.ClusterPartitionFetcher;
import org.apache.iotdb.db.mpp.plan.statement.Statement;
@@ -63,6 +63,9 @@ public class AuditLogger {
private static final SessionManager SESSION_MANAGER = SessionManager.getInstance();
+ private static final DataNodeDevicePathCache DEVICE_PATH_CACHE =
+ DataNodeDevicePathCache.getInstance();
+
private AuditLogger() {
// empty constructor
}
@@ -72,7 +75,7 @@ public class AuditLogger {
String log, String address, String username)
throws IoTDBConnectionException, IllegalPathException, QueryProcessException {
InsertRowStatement insertStatement = new InsertRowStatement();
- insertStatement.setDevicePath(new PartialPath(String.format(AUDIT_LOG_DEVICE, username)));
+ insertStatement.setDevicePath(DEVICE_PATH_CACHE.get(String.format(AUDIT_LOG_DEVICE, username)));
insertStatement.setTime(DateTimeUtils.currentTime());
insertStatement.setMeasurements(new String[] {LOG, USERNAME, ADDRESS});
insertStatement.setAligned(false);
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
index 432b76a718..bece2bba0b 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics;
import org.apache.iotdb.db.constant.SqlConstant;
import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.cache.DataNodeDevicePathCache;
import org.apache.iotdb.db.metadata.template.TemplateQueryType;
import org.apache.iotdb.db.metadata.utils.MetaFormatUtils;
import org.apache.iotdb.db.mpp.plan.expression.binary.GreaterEqualExpression;
@@ -120,6 +121,9 @@ public class StatementGenerator {
private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS =
PerformanceOverviewMetrics.getInstance();
+ private static final DataNodeDevicePathCache DEVICE_PATH_CACHE =
+ DataNodeDevicePathCache.getInstance();
+
public static Statement createStatement(String sql, ZoneId zoneId) {
return invokeParser(sql, zoneId);
}
@@ -256,7 +260,7 @@ public class StatementGenerator {
final long startTime = System.nanoTime();
// construct insert statement
InsertRowStatement insertStatement = new InsertRowStatement();
- insertStatement.setDevicePath(new PartialPath(insertRecordReq.getPrefixPath()));
+ insertStatement.setDevicePath(DEVICE_PATH_CACHE.get(insertRecordReq.getPrefixPath()));
insertStatement.setTime(insertRecordReq.getTimestamp());
insertStatement.setMeasurements(insertRecordReq.getMeasurements().toArray(new String[0]));
insertStatement.setAligned(insertRecordReq.isAligned);
@@ -270,7 +274,7 @@ public class StatementGenerator {
final long startTime = System.nanoTime();
// construct insert statement
InsertRowStatement insertStatement = new InsertRowStatement();
- insertStatement.setDevicePath(new PartialPath(insertRecordReq.getPrefixPath()));
+ insertStatement.setDevicePath(DEVICE_PATH_CACHE.get(insertRecordReq.getPrefixPath()));
insertStatement.setTime(insertRecordReq.getTimestamp());
insertStatement.setMeasurements(insertRecordReq.getMeasurements().toArray(new String[0]));
insertStatement.setDataTypes(new TSDataType[insertStatement.getMeasurements().length]);
@@ -286,7 +290,7 @@ public class StatementGenerator {
final long startTime = System.nanoTime();
// construct insert statement
InsertTabletStatement insertStatement = new InsertTabletStatement();
- insertStatement.setDevicePath(new PartialPath(insertTabletReq.getPrefixPath()));
+ insertStatement.setDevicePath(DEVICE_PATH_CACHE.get(insertTabletReq.getPrefixPath()));
insertStatement.setMeasurements(insertTabletReq.getMeasurements().toArray(new String[0]));
insertStatement.setTimes(
QueryDataSetUtils.readTimesFromBuffer(insertTabletReq.timestamps, insertTabletReq.size));
@@ -318,7 +322,7 @@ public class StatementGenerator {
List<InsertTabletStatement> insertTabletStatementList = new ArrayList<>();
for (int i = 0; i < req.prefixPaths.size(); i++) {
InsertTabletStatement insertTabletStatement = new InsertTabletStatement();
- insertTabletStatement.setDevicePath(new PartialPath(req.prefixPaths.get(i)));
+ insertTabletStatement.setDevicePath(DEVICE_PATH_CACHE.get(req.prefixPaths.get(i)));
insertTabletStatement.setMeasurements(req.measurementsList.get(i).toArray(new String[0]));
insertTabletStatement.setTimes(
QueryDataSetUtils.readTimesFromBuffer(req.timestampsList.get(i), req.sizeList.get(i)));
@@ -357,7 +361,7 @@ public class StatementGenerator {
List<InsertRowStatement> insertRowStatementList = new ArrayList<>();
for (int i = 0; i < req.prefixPaths.size(); i++) {
InsertRowStatement statement = new InsertRowStatement();
- statement.setDevicePath(new PartialPath(req.getPrefixPaths().get(i)));
+ statement.setDevicePath(DEVICE_PATH_CACHE.get(req.getPrefixPaths().get(i)));
statement.setMeasurements(req.getMeasurementsList().get(i).toArray(new String[0]));
statement.setTime(req.getTimestamps().get(i));
statement.fillValues(req.valuesList.get(i));
@@ -381,7 +385,7 @@ public class StatementGenerator {
List<InsertRowStatement> insertRowStatementList = new ArrayList<>();
for (int i = 0; i < req.prefixPaths.size(); i++) {
InsertRowStatement statement = new InsertRowStatement();
- statement.setDevicePath(new PartialPath(req.getPrefixPaths().get(i)));
+ statement.setDevicePath(DEVICE_PATH_CACHE.get(req.getPrefixPaths().get(i)));
addMeasurementAndValue(
statement, req.getMeasurementsList().get(i), req.getValuesList().get(i));
statement.setDataTypes(new TSDataType[statement.getMeasurements().length]);
@@ -404,7 +408,7 @@ public class StatementGenerator {
final long startTime = System.nanoTime();
// construct insert statement
InsertRowsOfOneDeviceStatement insertStatement = new InsertRowsOfOneDeviceStatement();
- insertStatement.setDevicePath(new PartialPath(req.prefixPath));
+ insertStatement.setDevicePath(DEVICE_PATH_CACHE.get(req.prefixPath));
List<InsertRowStatement> insertRowStatementList = new ArrayList<>();
for (int i = 0; i < req.timestamps.size(); i++) {
InsertRowStatement statement = new InsertRowStatement();
@@ -429,7 +433,7 @@ public class StatementGenerator {
final long startTime = System.nanoTime();
// construct insert statement
InsertRowsOfOneDeviceStatement insertStatement = new InsertRowsOfOneDeviceStatement();
- insertStatement.setDevicePath(new PartialPath(req.prefixPath));
+ insertStatement.setDevicePath(DEVICE_PATH_CACHE.get(req.prefixPath));
List<InsertRowStatement> insertRowStatementList = new ArrayList<>();
for (int i = 0; i < req.timestamps.size(); i++) {
InsertRowStatement statement = new InsertRowStatement();
@@ -820,7 +824,7 @@ public class StatementGenerator {
+ TsFileConstant.PATH_SEPARATOR
+ recordModelMetricsReq.getTrialId();
InsertRowStatement insertRowStatement = new InsertRowStatement();
- insertRowStatement.setDevicePath(new PartialPath(path));
+ insertRowStatement.setDevicePath(DEVICE_PATH_CACHE.get(path));
insertRowStatement.setTime(recordModelMetricsReq.getTimestamp());
insertRowStatement.setMeasurements(recordModelMetricsReq.getMetrics().toArray(new String[0]));
insertRowStatement.setAligned(true);
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java
index 7e8cae62b1..5d12a6d458 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java
@@ -19,10 +19,10 @@ package org.apache.iotdb.db.protocol.mqtt;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.conf.IoTDBConstant.ClientVersion;
-import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.metadata.cache.DataNodeDevicePathCache;
import org.apache.iotdb.db.mpp.plan.Coordinator;
import org.apache.iotdb.db.mpp.plan.analyze.ClusterPartitionFetcher;
import org.apache.iotdb.db.mpp.plan.analyze.IPartitionFetcher;
@@ -136,7 +136,7 @@ public class MPPPublishHandler extends AbstractInterceptHandler {
TSStatus tsStatus = null;
try {
InsertRowStatement statement = new InsertRowStatement();
- statement.setDevicePath(new PartialPath(event.getDevice()));
+ statement.setDevicePath(DataNodeDevicePathCache.getInstance().get(event.getDevice()));
statement.setTime(event.getTimestamp());
statement.setMeasurements(event.getMeasurements().toArray(new String[0]));
if (event.getDataTypes() == null) {
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java
index c0d422601d..3f488ee7bc 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java
@@ -18,8 +18,8 @@
package org.apache.iotdb.db.protocol.rest.v1.handler;
import org.apache.iotdb.commons.exception.IllegalPathException;
-import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.exception.WriteProcessRejectException;
+import org.apache.iotdb.db.metadata.cache.DataNodeDevicePathCache;
import org.apache.iotdb.db.mpp.plan.statement.crud.InsertTabletStatement;
import org.apache.iotdb.db.protocol.rest.v1.model.InsertTabletRequest;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -38,7 +38,8 @@ public class StatementConstructionHandler {
throws IllegalPathException, WriteProcessRejectException {
// construct insert statement
InsertTabletStatement insertStatement = new InsertTabletStatement();
- insertStatement.setDevicePath(new PartialPath(insertTabletRequest.getDeviceId()));
+ insertStatement.setDevicePath(
+ DataNodeDevicePathCache.getInstance().get(insertTabletRequest.getDeviceId()));
insertStatement.setMeasurements(insertTabletRequest.getMeasurements().toArray(new String[0]));
List<List<Object>> rawData = insertTabletRequest.getValues();
List<String> rawDataType = insertTabletRequest.getDataTypes();
diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
index 4d643862d9..850210a725 100644
--- a/server/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
+++ b/server/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
@@ -18,8 +18,8 @@
package org.apache.iotdb.db.protocol.rest.v2.handler;
import org.apache.iotdb.commons.exception.IllegalPathException;
-import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.exception.WriteProcessRejectException;
+import org.apache.iotdb.db.metadata.cache.DataNodeDevicePathCache;
import org.apache.iotdb.db.mpp.plan.statement.crud.InsertTabletStatement;
import org.apache.iotdb.db.protocol.rest.v2.model.InsertTabletRequest;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -38,7 +38,8 @@ public class StatementConstructionHandler {
throws IllegalPathException, WriteProcessRejectException {
// construct insert statement
InsertTabletStatement insertStatement = new InsertTabletStatement();
- insertStatement.setDevicePath(new PartialPath(insertTabletRequest.getDevice()));
+ insertStatement.setDevicePath(
+ DataNodeDevicePathCache.getInstance().get(insertTabletRequest.getDevice()));
insertStatement.setMeasurements(insertTabletRequest.getMeasurements().toArray(new String[0]));
List<List<Object>> rawData = insertTabletRequest.getValues();
List<String> rawDataType = insertTabletRequest.getDataTypes();
diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 34db75c702..165a64b5ad 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -60,6 +60,7 @@ import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.settle.SettleRequestHandler;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.metadata.cache.DataNodeDevicePathCache;
import org.apache.iotdb.db.metadata.cache.DataNodeSchemaCache;
import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion;
import org.apache.iotdb.db.metadata.schemaregion.SchemaEngine;
@@ -1542,6 +1543,7 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
// TODO what need to clean?
ClusterPartitionFetcher.getInstance().invalidAllCache();
DataNodeSchemaCache.getInstance().cleanUp();
+ DataNodeDevicePathCache.getInstance().cleanUp();
return status;
}