You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/05/25 08:25:55 UTC
[iotdb] 01/01: [IOTDB-3242] Avoid generating same QueryId in different DataNode
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch QueryIdGenerator
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 44c36a92a971c0f098dedd9593329721a1fd3683
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Wed May 25 16:25:36 2022 +0800
[IOTDB-3242] Avoid generating same QueryId in different DataNode
---
.../iotdb/db/mpp/execution/QueryIdGenerator.java | 119 +++++++++++++++++++++
.../org/apache/iotdb/db/mpp/plan/Coordinator.java | 15 ++-
.../db/mpp/plan/analyze/ClusterSchemaFetcher.java | 10 +-
.../thrift/impl/DataNodeTSIServiceImpl.java | 41 ++++---
.../service/thrift/impl/InternalServiceImpl.java | 5 +-
5 files changed, 154 insertions(+), 36 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/QueryIdGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/QueryIdGenerator.java
new file mode 100644
index 0000000000..ffaa2fe4dd
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/QueryIdGenerator.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.mpp.execution;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.mpp.common.QueryId;
+
+import javax.annotation.concurrent.GuardedBy;
+
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static java.lang.String.format;
+import static java.time.ZoneOffset.UTC;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+public class QueryIdGenerator {
+
+ private static final DateTimeFormatter TIMESTAMP_FORMAT =
+ DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss").withZone(UTC);
+ private static final long BASE_SYSTEM_TIME_MILLIS = System.currentTimeMillis();
+ private static final long BASE_NANO_TIME = System.nanoTime();
+
+ private final String dataNodeId;
+
+ @GuardedBy("this")
+ private long lastTimeInDays;
+
+ @GuardedBy("this")
+ private long lastTimeInSeconds;
+
+ @GuardedBy("this")
+ private String lastTimestamp;
+
+ @GuardedBy("this")
+ private int counter;
+
+ public QueryIdGenerator() {
+ this.dataNodeId = String.valueOf(IoTDBDescriptor.getInstance().getConfig().getDataNodeId());
+ }
+
+ public String getCoordinatorId() {
+ return dataNodeId;
+ }
+
+ /**
+ * Generate next queryId using the following format: {@code YYYYMMDD_hhmmss_index_dataNodeId}
+ *
+ * <p>{@code index} rolls at the start of every day or when it is close to reaching {@code
+ * 99,999}. {@code dataNodeId} is a unique id generated by config node when this data node is
+ * created.
+ */
+ public synchronized QueryId createNextQueryId() {
+ long now = nowInMillis();
+
+ // restart counter if it is at the limit
+ if (counter > 99_999) {
+ // Wait for the timestamp to change to reset the counter to avoid duplicates
+ // in the very unlikely case we generated all 100,000 in the previous second
+ while (MILLISECONDS.toSeconds(now) == lastTimeInSeconds) {
+ long delta = SECONDS.toMillis(lastTimeInSeconds + 1) - now;
+ sleepUninterruptibly(delta, MILLISECONDS);
+ now = nowInMillis();
+ }
+ counter = 0;
+ }
+
+ // if the second changed since the last ID was generated, generate a new timestamp
+ if (MILLISECONDS.toSeconds(now) != lastTimeInSeconds) {
+ // generate new timestamp
+ lastTimeInSeconds = MILLISECONDS.toSeconds(now);
+ lastTimestamp = formatEpochMilli(now);
+
+ // if the day has rolled over, restart the counter
+ if (MILLISECONDS.toDays(now) != lastTimeInDays) {
+ lastTimeInDays = MILLISECONDS.toDays(now);
+ counter = 0;
+ }
+
+ // restart the counter if it is close to the limit, to avoid sleep on rollover
+ if (counter >= 90_000) {
+ counter = 0;
+ }
+ }
+
+ long index = counter;
+ counter++;
+
+ return new QueryId(format("%s_%05d_%s", lastTimestamp, index, dataNodeId));
+ }
+
+ private static String formatEpochMilli(long milli) {
+ return TIMESTAMP_FORMAT.format(Instant.ofEpochMilli(milli));
+ }
+
+ protected long nowInMillis() {
+ // avoid problems with the clock moving backwards
+ return BASE_SYSTEM_TIME_MILLIS + NANOSECONDS.toMillis(System.nanoTime() - BASE_NANO_TIME);
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/Coordinator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/Coordinator.java
index d56457a7ce..cf19379297 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/Coordinator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/Coordinator.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.mpp.common.MPPQueryContext;
import org.apache.iotdb.db.mpp.common.QueryId;
import org.apache.iotdb.db.mpp.common.SessionInfo;
+import org.apache.iotdb.db.mpp.execution.QueryIdGenerator;
import org.apache.iotdb.db.mpp.plan.analyze.IPartitionFetcher;
import org.apache.iotdb.db.mpp.plan.analyze.ISchemaFetcher;
import org.apache.iotdb.db.mpp.plan.execution.ExecutionResult;
@@ -84,9 +85,11 @@ public class Coordinator {
private final ExecutorService writeOperationExecutor;
private final ScheduledExecutorService scheduledExecutor;
+ private final QueryIdGenerator queryIdGenerator = new QueryIdGenerator();
+
private static final Coordinator INSTANCE = new Coordinator();
- private final ConcurrentHashMap<QueryId, IQueryExecution> queryExecutionMap;
+ private final ConcurrentHashMap<Long, IQueryExecution> queryExecutionMap;
private Coordinator() {
this.queryExecutionMap = new ConcurrentHashMap<>();
@@ -117,7 +120,7 @@ public class Coordinator {
public ExecutionResult execute(
Statement statement,
- QueryId queryId,
+ long queryId,
SessionInfo session,
String sql,
IPartitionFetcher partitionFetcher,
@@ -128,7 +131,7 @@ public class Coordinator {
statement,
new MPPQueryContext(
sql,
- queryId,
+ queryIdGenerator.createNextQueryId(),
session,
LOCAL_HOST_DATA_BLOCK_ENDPOINT,
LOCAL_HOST_INTERNAL_ENDPOINT),
@@ -140,7 +143,7 @@ public class Coordinator {
return execution.getStatus();
}
- public IQueryExecution getQueryExecution(QueryId queryId) {
+ public IQueryExecution getQueryExecution(Long queryId) {
return queryExecutionMap.get(queryId);
}
@@ -161,6 +164,10 @@ public class Coordinator {
COORDINATOR_SCHEDULED_EXECUTOR_SIZE, COORDINATOR_SCHEDULED_EXECUTOR_NAME);
}
+ public QueryId createQueryId() {
+ return queryIdGenerator.createNextQueryId();
+ }
+
public static Coordinator getInstance() {
return INSTANCE;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java
index 8f561fc1d7..49b57c79b5 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java
@@ -26,7 +26,6 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.metadata.cache.DataNodeSchemaCache;
-import org.apache.iotdb.db.mpp.common.QueryId;
import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo;
import org.apache.iotdb.db.mpp.common.schematree.PathPatternTree;
import org.apache.iotdb.db.mpp.common.schematree.SchemaTree;
@@ -97,8 +96,7 @@ public class ClusterSchemaFetcher implements ISchemaFetcher {
private SchemaTree executeSchemaFetchQuery(SchemaFetchStatement schemaFetchStatement) {
- QueryId queryId =
- new QueryId(String.valueOf(SessionManager.getInstance().requestQueryId(false)));
+ long queryId = SessionManager.getInstance().requestQueryId(false);
ExecutionResult executionResult =
coordinator.execute(schemaFetchStatement, queryId, null, "", partitionFetcher, this);
// TODO: (xingtanzjr) throw exception
@@ -327,8 +325,7 @@ public class ClusterSchemaFetcher implements ISchemaFetcher {
}
private void executeCreateStatement(Statement statement) {
- QueryId queryId =
- new QueryId(String.valueOf(SessionManager.getInstance().requestQueryId(false)));
+ long queryId = SessionManager.getInstance().requestQueryId(false);
ExecutionResult executionResult =
coordinator.execute(statement, queryId, null, "", partitionFetcher, this);
// TODO: throw exception
@@ -346,8 +343,7 @@ public class ClusterSchemaFetcher implements ISchemaFetcher {
private void executeCreateTimeseriesByDeviceStatement(
CreateTimeSeriesByDeviceStatement statement) {
- QueryId queryId =
- new QueryId(String.valueOf(SessionManager.getInstance().requestQueryId(false)));
+ long queryId = SessionManager.getInstance().requestQueryId(false);
ExecutionResult executionResult =
coordinator.execute(statement, queryId, null, "", partitionFetcher, this);
// TODO: throw exception
diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeTSIServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeTSIServiceImpl.java
index 2823642630..6f059e886f 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeTSIServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeTSIServiceImpl.java
@@ -306,7 +306,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
ExecutionResult result =
COORDINATOR.execute(
statement,
- new QueryId(String.valueOf(queryId)),
+ queryId,
SESSION_MANAGER.getSessionInfo(sessionId),
"",
PARTITION_FETCHER,
@@ -348,7 +348,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
ExecutionResult result =
COORDINATOR.execute(
statement,
- new QueryId(String.valueOf(queryId)),
+ queryId,
SESSION_MANAGER.getSessionInfo(req.sessionId),
"",
PARTITION_FETCHER,
@@ -409,7 +409,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
ExecutionResult result =
COORDINATOR.execute(
statement,
- new QueryId(String.valueOf(queryId)),
+ queryId,
SESSION_MANAGER.getSessionInfo(req.sessionId),
"",
PARTITION_FETCHER,
@@ -457,7 +457,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
ExecutionResult result =
COORDINATOR.execute(
statement,
- new QueryId(String.valueOf(queryId)),
+ queryId,
SESSION_MANAGER.getSessionInfo(req.sessionId),
"",
PARTITION_FETCHER,
@@ -507,7 +507,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
ExecutionResult result =
COORDINATOR.execute(
statement,
- new QueryId(String.valueOf(queryId)),
+ queryId,
SESSION_MANAGER.getSessionInfo(sessionId),
"",
PARTITION_FETCHER,
@@ -548,12 +548,11 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
try {
long queryId = SESSION_MANAGER.requestQueryId(req.statementId, true);
- QueryId id = new QueryId(String.valueOf(queryId));
// create and cache dataset
ExecutionResult result =
COORDINATOR.execute(
s,
- id,
+ queryId,
SESSION_MANAGER.getSessionInfo(req.sessionId),
statement,
PARTITION_FETCHER,
@@ -563,7 +562,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
throw new RuntimeException("error code: " + result.status);
}
- IQueryExecution queryExecution = COORDINATOR.getQueryExecution(id);
+ IQueryExecution queryExecution = COORDINATOR.getQueryExecution(queryId);
TSExecuteStatementResp resp;
if (queryExecution.isQuery()) {
@@ -615,8 +614,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
TSFetchResultsResp resp = RpcUtils.getTSFetchResultsResp(TSStatusCode.SUCCESS_STATUS);
TSQueryDataSet result =
QueryDataSetUtils.convertTsBlockByFetchSize(
- COORDINATOR.getQueryExecution(new QueryId(String.valueOf(req.queryId))),
- req.fetchSize);
+ COORDINATOR.getQueryExecution(req.queryId), req.fetchSize);
boolean hasResultSet = result.bufferForTime().limit() != 0;
resp.setHasResultSet(hasResultSet);
@@ -664,7 +662,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
ExecutionResult result =
COORDINATOR.execute(
statement,
- new QueryId(String.valueOf(queryId)),
+ queryId,
SESSION_MANAGER.getSessionInfo(req.sessionId),
"",
PARTITION_FETCHER,
@@ -715,7 +713,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
ExecutionResult result =
COORDINATOR.execute(
statement,
- new QueryId(String.valueOf(queryId)),
+ queryId,
SESSION_MANAGER.getSessionInfo(req.sessionId),
"",
PARTITION_FETCHER,
@@ -766,7 +764,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
ExecutionResult result =
COORDINATOR.execute(
statement,
- new QueryId(String.valueOf(queryId)),
+ queryId,
SESSION_MANAGER.getSessionInfo(req.sessionId),
"",
PARTITION_FETCHER,
@@ -815,7 +813,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
ExecutionResult result =
COORDINATOR.execute(
statement,
- new QueryId(String.valueOf(queryId)),
+ queryId,
SESSION_MANAGER.getSessionInfo(req.sessionId),
"",
PARTITION_FETCHER,
@@ -857,7 +855,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
ExecutionResult result =
COORDINATOR.execute(
statement,
- new QueryId(String.valueOf(queryId)),
+ queryId,
SESSION_MANAGER.getSessionInfo(req.sessionId),
"",
PARTITION_FETCHER,
@@ -899,7 +897,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
ExecutionResult result =
COORDINATOR.execute(
statement,
- new QueryId(String.valueOf(queryId)),
+ queryId,
SESSION_MANAGER.getSessionInfo(req.sessionId),
"",
PARTITION_FETCHER,
@@ -947,7 +945,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
ExecutionResult result =
COORDINATOR.execute(
statement,
- genQueryId(queryId),
+ queryId,
SESSION_MANAGER.getSessionInfo(req.sessionId),
"",
PARTITION_FETCHER,
@@ -1030,12 +1028,11 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
QUERY_FREQUENCY_RECORDER.incrementAndGet();
AUDIT_LOGGER.debug("Session {} execute Row Data Query: {}", req.sessionId, req);
long queryId = SESSION_MANAGER.requestQueryId(req.statementId, true);
- QueryId id = new QueryId(String.valueOf(queryId));
// create and cache dataset
ExecutionResult result =
COORDINATOR.execute(
s,
- id,
+ queryId,
SESSION_MANAGER.getSessionInfo(req.sessionId),
"",
PARTITION_FETCHER,
@@ -1045,7 +1042,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
throw new RuntimeException("error code: " + result.status);
}
- IQueryExecution queryExecution = COORDINATOR.getQueryExecution(id);
+ IQueryExecution queryExecution = COORDINATOR.getQueryExecution(queryId);
TSExecuteStatementResp resp;
if (queryExecution.isQuery()) {
@@ -1148,7 +1145,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
ExecutionResult result =
COORDINATOR.execute(
statement,
- new QueryId(String.valueOf(queryId)),
+ queryId,
SESSION_MANAGER.getSessionInfo(req.sessionId),
"",
PARTITION_FETCHER,
@@ -1212,7 +1209,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
}
private void cleanupQueryExecution(Long queryId) {
- IQueryExecution queryExecution = COORDINATOR.getQueryExecution(genQueryId(queryId));
+ IQueryExecution queryExecution = COORDINATOR.getQueryExecution(queryId);
if (queryExecution != null) {
queryExecution.stopAndCleanup();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InternalServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InternalServiceImpl.java
index 4c7f08edf0..fb9480f272 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InternalServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InternalServiceImpl.java
@@ -50,6 +50,7 @@ import org.apache.iotdb.db.mpp.common.PlanFragmentId;
import org.apache.iotdb.db.mpp.common.QueryId;
import org.apache.iotdb.db.mpp.execution.fragment.FragmentInstanceInfo;
import org.apache.iotdb.db.mpp.execution.fragment.FragmentInstanceManager;
+import org.apache.iotdb.db.mpp.plan.Coordinator;
import org.apache.iotdb.db.mpp.plan.analyze.ClusterPartitionFetcher;
import org.apache.iotdb.db.mpp.plan.analyze.QueryType;
import org.apache.iotdb.db.mpp.plan.analyze.SchemaValidator;
@@ -59,7 +60,6 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.DeleteRegionNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode;
-import org.apache.iotdb.db.query.control.SessionManager;
import org.apache.iotdb.db.service.metrics.MetricsService;
import org.apache.iotdb.db.service.metrics.enums.Metric;
import org.apache.iotdb.db.service.metrics.enums.Tag;
@@ -338,8 +338,7 @@ public class InternalServiceImpl implements InternalService.Iface {
@Override
public TSStatus deleteRegion(TConsensusGroupId tconsensusGroupId) throws TException {
- long queryIdRaw = SessionManager.getInstance().requestQueryId(false);
- QueryId queryId = new QueryId(String.valueOf(queryIdRaw));
+ QueryId queryId = Coordinator.getInstance().createQueryId();
PlanNodeId planNodeId = queryId.genPlanNodeId();
DeleteRegionNode deleteRegionNode = new DeleteRegionNode(queryId.genPlanNodeId());
ConsensusGroupId consensusGroupId =