You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/05/25 08:25:55 UTC

[iotdb] 01/01: [IOTDB-3242] Avoid generating same QueryId in different DataNode

This is an automated email from the ASF dual-hosted git repository.

jackietien pushed a commit to branch QueryIdGenerator
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 44c36a92a971c0f098dedd9593329721a1fd3683
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Wed May 25 16:25:36 2022 +0800

    [IOTDB-3242] Avoid generating same QueryId in different DataNode
---
 .../iotdb/db/mpp/execution/QueryIdGenerator.java   | 119 +++++++++++++++++++++
 .../org/apache/iotdb/db/mpp/plan/Coordinator.java  |  15 ++-
 .../db/mpp/plan/analyze/ClusterSchemaFetcher.java  |  10 +-
 .../thrift/impl/DataNodeTSIServiceImpl.java        |  41 ++++---
 .../service/thrift/impl/InternalServiceImpl.java   |   5 +-
 5 files changed, 154 insertions(+), 36 deletions(-)

diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/QueryIdGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/QueryIdGenerator.java
new file mode 100644
index 0000000000..ffaa2fe4dd
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/QueryIdGenerator.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.mpp.execution;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.mpp.common.QueryId;
+
+import javax.annotation.concurrent.GuardedBy;
+
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static java.lang.String.format;
+import static java.time.ZoneOffset.UTC;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+public class QueryIdGenerator {
+
+  private static final DateTimeFormatter TIMESTAMP_FORMAT =
+      DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss").withZone(UTC);
+  private static final long BASE_SYSTEM_TIME_MILLIS = System.currentTimeMillis();
+  private static final long BASE_NANO_TIME = System.nanoTime();
+
+  private final String dataNodeId;
+
+  @GuardedBy("this")
+  private long lastTimeInDays;
+
+  @GuardedBy("this")
+  private long lastTimeInSeconds;
+
+  @GuardedBy("this")
+  private String lastTimestamp;
+
+  @GuardedBy("this")
+  private int counter;
+
+  public QueryIdGenerator() {
+    this.dataNodeId = String.valueOf(IoTDBDescriptor.getInstance().getConfig().getDataNodeId());
+  }
+
+  public String getCoordinatorId() {
+    return dataNodeId;
+  }
+
+  /**
+   * Generate next queryId using the following format: {@code YYYYMMDD_hhmmss_index_dataNodeId}
+   *
+   * <p>{@code index} rolls at the start of every day or when it is close to reaching {@code
+   * 99,999}. {@code dataNodeId} is a unique id generated by config node when this data node is
+   * created.
+   */
+  public synchronized QueryId createNextQueryId() {
+    long now = nowInMillis();
+
+    // restart counter if it is at the limit
+    if (counter > 99_999) {
+      // Wait for the timestamp to change to reset the counter to avoid duplicates
+      // in the very unlikely case we generated all 100,000 in the previous second
+      while (MILLISECONDS.toSeconds(now) == lastTimeInSeconds) {
+        long delta = SECONDS.toMillis(lastTimeInSeconds + 1) - now;
+        sleepUninterruptibly(delta, MILLISECONDS);
+        now = nowInMillis();
+      }
+      counter = 0;
+    }
+
+    // if the second changed since the last ID was generated, generate a new timestamp
+    if (MILLISECONDS.toSeconds(now) != lastTimeInSeconds) {
+      // generate new timestamp
+      lastTimeInSeconds = MILLISECONDS.toSeconds(now);
+      lastTimestamp = formatEpochMilli(now);
+
+      // if the day has rolled over, restart the counter
+      if (MILLISECONDS.toDays(now) != lastTimeInDays) {
+        lastTimeInDays = MILLISECONDS.toDays(now);
+        counter = 0;
+      }
+
+      // restart the counter if it is close to the limit, to avoid sleep on rollover
+      if (counter >= 90_000) {
+        counter = 0;
+      }
+    }
+
+    long index = counter;
+    counter++;
+
+    return new QueryId(format("%s_%05d_%s", lastTimestamp, index, dataNodeId));
+  }
+
+  private static String formatEpochMilli(long milli) {
+    return TIMESTAMP_FORMAT.format(Instant.ofEpochMilli(milli));
+  }
+
+  protected long nowInMillis() {
+    // avoid problems with the clock moving backwards
+    return BASE_SYSTEM_TIME_MILLIS + NANOSECONDS.toMillis(System.nanoTime() - BASE_NANO_TIME);
+  }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/Coordinator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/Coordinator.java
index d56457a7ce..cf19379297 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/Coordinator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/Coordinator.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.mpp.common.MPPQueryContext;
 import org.apache.iotdb.db.mpp.common.QueryId;
 import org.apache.iotdb.db.mpp.common.SessionInfo;
+import org.apache.iotdb.db.mpp.execution.QueryIdGenerator;
 import org.apache.iotdb.db.mpp.plan.analyze.IPartitionFetcher;
 import org.apache.iotdb.db.mpp.plan.analyze.ISchemaFetcher;
 import org.apache.iotdb.db.mpp.plan.execution.ExecutionResult;
@@ -84,9 +85,11 @@ public class Coordinator {
   private final ExecutorService writeOperationExecutor;
   private final ScheduledExecutorService scheduledExecutor;
 
+  private final QueryIdGenerator queryIdGenerator = new QueryIdGenerator();
+
   private static final Coordinator INSTANCE = new Coordinator();
 
-  private final ConcurrentHashMap<QueryId, IQueryExecution> queryExecutionMap;
+  private final ConcurrentHashMap<Long, IQueryExecution> queryExecutionMap;
 
   private Coordinator() {
     this.queryExecutionMap = new ConcurrentHashMap<>();
@@ -117,7 +120,7 @@ public class Coordinator {
 
   public ExecutionResult execute(
       Statement statement,
-      QueryId queryId,
+      long queryId,
       SessionInfo session,
       String sql,
       IPartitionFetcher partitionFetcher,
@@ -128,7 +131,7 @@ public class Coordinator {
             statement,
             new MPPQueryContext(
                 sql,
-                queryId,
+                queryIdGenerator.createNextQueryId(),
                 session,
                 LOCAL_HOST_DATA_BLOCK_ENDPOINT,
                 LOCAL_HOST_INTERNAL_ENDPOINT),
@@ -140,7 +143,7 @@ public class Coordinator {
     return execution.getStatus();
   }
 
-  public IQueryExecution getQueryExecution(QueryId queryId) {
+  public IQueryExecution getQueryExecution(Long queryId) {
     return queryExecutionMap.get(queryId);
   }
 
@@ -161,6 +164,10 @@ public class Coordinator {
         COORDINATOR_SCHEDULED_EXECUTOR_SIZE, COORDINATOR_SCHEDULED_EXECUTOR_NAME);
   }
 
+  public QueryId createQueryId() {
+    return queryIdGenerator.createNextQueryId();
+  }
+
   public static Coordinator getInstance() {
     return INSTANCE;
   }
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java
index 8f561fc1d7..49b57c79b5 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterSchemaFetcher.java
@@ -26,7 +26,6 @@ import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.db.conf.IoTDBConfig;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.metadata.cache.DataNodeSchemaCache;
-import org.apache.iotdb.db.mpp.common.QueryId;
 import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo;
 import org.apache.iotdb.db.mpp.common.schematree.PathPatternTree;
 import org.apache.iotdb.db.mpp.common.schematree.SchemaTree;
@@ -97,8 +96,7 @@ public class ClusterSchemaFetcher implements ISchemaFetcher {
 
   private SchemaTree executeSchemaFetchQuery(SchemaFetchStatement schemaFetchStatement) {
 
-    QueryId queryId =
-        new QueryId(String.valueOf(SessionManager.getInstance().requestQueryId(false)));
+    long queryId = SessionManager.getInstance().requestQueryId(false);
     ExecutionResult executionResult =
         coordinator.execute(schemaFetchStatement, queryId, null, "", partitionFetcher, this);
     // TODO: (xingtanzjr) throw exception
@@ -327,8 +325,7 @@ public class ClusterSchemaFetcher implements ISchemaFetcher {
   }
 
   private void executeCreateStatement(Statement statement) {
-    QueryId queryId =
-        new QueryId(String.valueOf(SessionManager.getInstance().requestQueryId(false)));
+    long queryId = SessionManager.getInstance().requestQueryId(false);
     ExecutionResult executionResult =
         coordinator.execute(statement, queryId, null, "", partitionFetcher, this);
     // TODO: throw exception
@@ -346,8 +343,7 @@ public class ClusterSchemaFetcher implements ISchemaFetcher {
 
   private void executeCreateTimeseriesByDeviceStatement(
       CreateTimeSeriesByDeviceStatement statement) {
-    QueryId queryId =
-        new QueryId(String.valueOf(SessionManager.getInstance().requestQueryId(false)));
+    long queryId = SessionManager.getInstance().requestQueryId(false);
     ExecutionResult executionResult =
         coordinator.execute(statement, queryId, null, "", partitionFetcher, this);
     // TODO: throw exception
diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeTSIServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeTSIServiceImpl.java
index 2823642630..6f059e886f 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeTSIServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeTSIServiceImpl.java
@@ -306,7 +306,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       ExecutionResult result =
           COORDINATOR.execute(
               statement,
-              new QueryId(String.valueOf(queryId)),
+              queryId,
               SESSION_MANAGER.getSessionInfo(sessionId),
               "",
               PARTITION_FETCHER,
@@ -348,7 +348,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       ExecutionResult result =
           COORDINATOR.execute(
               statement,
-              new QueryId(String.valueOf(queryId)),
+              queryId,
               SESSION_MANAGER.getSessionInfo(req.sessionId),
               "",
               PARTITION_FETCHER,
@@ -409,7 +409,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       ExecutionResult result =
           COORDINATOR.execute(
               statement,
-              new QueryId(String.valueOf(queryId)),
+              queryId,
               SESSION_MANAGER.getSessionInfo(req.sessionId),
               "",
               PARTITION_FETCHER,
@@ -457,7 +457,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       ExecutionResult result =
           COORDINATOR.execute(
               statement,
-              new QueryId(String.valueOf(queryId)),
+              queryId,
               SESSION_MANAGER.getSessionInfo(req.sessionId),
               "",
               PARTITION_FETCHER,
@@ -507,7 +507,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       ExecutionResult result =
           COORDINATOR.execute(
               statement,
-              new QueryId(String.valueOf(queryId)),
+              queryId,
               SESSION_MANAGER.getSessionInfo(sessionId),
               "",
               PARTITION_FETCHER,
@@ -548,12 +548,11 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
 
     try {
       long queryId = SESSION_MANAGER.requestQueryId(req.statementId, true);
-      QueryId id = new QueryId(String.valueOf(queryId));
       // create and cache dataset
       ExecutionResult result =
           COORDINATOR.execute(
               s,
-              id,
+              queryId,
               SESSION_MANAGER.getSessionInfo(req.sessionId),
               statement,
               PARTITION_FETCHER,
@@ -563,7 +562,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
         throw new RuntimeException("error code: " + result.status);
       }
 
-      IQueryExecution queryExecution = COORDINATOR.getQueryExecution(id);
+      IQueryExecution queryExecution = COORDINATOR.getQueryExecution(queryId);
 
       TSExecuteStatementResp resp;
       if (queryExecution.isQuery()) {
@@ -615,8 +614,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       TSFetchResultsResp resp = RpcUtils.getTSFetchResultsResp(TSStatusCode.SUCCESS_STATUS);
       TSQueryDataSet result =
           QueryDataSetUtils.convertTsBlockByFetchSize(
-              COORDINATOR.getQueryExecution(new QueryId(String.valueOf(req.queryId))),
-              req.fetchSize);
+              COORDINATOR.getQueryExecution(req.queryId), req.fetchSize);
       boolean hasResultSet = result.bufferForTime().limit() != 0;
 
       resp.setHasResultSet(hasResultSet);
@@ -664,7 +662,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       ExecutionResult result =
           COORDINATOR.execute(
               statement,
-              new QueryId(String.valueOf(queryId)),
+              queryId,
               SESSION_MANAGER.getSessionInfo(req.sessionId),
               "",
               PARTITION_FETCHER,
@@ -715,7 +713,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       ExecutionResult result =
           COORDINATOR.execute(
               statement,
-              new QueryId(String.valueOf(queryId)),
+              queryId,
               SESSION_MANAGER.getSessionInfo(req.sessionId),
               "",
               PARTITION_FETCHER,
@@ -766,7 +764,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       ExecutionResult result =
           COORDINATOR.execute(
               statement,
-              new QueryId(String.valueOf(queryId)),
+              queryId,
               SESSION_MANAGER.getSessionInfo(req.sessionId),
               "",
               PARTITION_FETCHER,
@@ -815,7 +813,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       ExecutionResult result =
           COORDINATOR.execute(
               statement,
-              new QueryId(String.valueOf(queryId)),
+              queryId,
               SESSION_MANAGER.getSessionInfo(req.sessionId),
               "",
               PARTITION_FETCHER,
@@ -857,7 +855,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       ExecutionResult result =
           COORDINATOR.execute(
               statement,
-              new QueryId(String.valueOf(queryId)),
+              queryId,
               SESSION_MANAGER.getSessionInfo(req.sessionId),
               "",
               PARTITION_FETCHER,
@@ -899,7 +897,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       ExecutionResult result =
           COORDINATOR.execute(
               statement,
-              new QueryId(String.valueOf(queryId)),
+              queryId,
               SESSION_MANAGER.getSessionInfo(req.sessionId),
               "",
               PARTITION_FETCHER,
@@ -947,7 +945,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       ExecutionResult result =
           COORDINATOR.execute(
               statement,
-              genQueryId(queryId),
+              queryId,
               SESSION_MANAGER.getSessionInfo(req.sessionId),
               "",
               PARTITION_FETCHER,
@@ -1030,12 +1028,11 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       QUERY_FREQUENCY_RECORDER.incrementAndGet();
       AUDIT_LOGGER.debug("Session {} execute Row Data Query: {}", req.sessionId, req);
       long queryId = SESSION_MANAGER.requestQueryId(req.statementId, true);
-      QueryId id = new QueryId(String.valueOf(queryId));
       // create and cache dataset
       ExecutionResult result =
           COORDINATOR.execute(
               s,
-              id,
+              queryId,
               SESSION_MANAGER.getSessionInfo(req.sessionId),
               "",
               PARTITION_FETCHER,
@@ -1045,7 +1042,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
         throw new RuntimeException("error code: " + result.status);
       }
 
-      IQueryExecution queryExecution = COORDINATOR.getQueryExecution(id);
+      IQueryExecution queryExecution = COORDINATOR.getQueryExecution(queryId);
 
       TSExecuteStatementResp resp;
       if (queryExecution.isQuery()) {
@@ -1148,7 +1145,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
       ExecutionResult result =
           COORDINATOR.execute(
               statement,
-              new QueryId(String.valueOf(queryId)),
+              queryId,
               SESSION_MANAGER.getSessionInfo(req.sessionId),
               "",
               PARTITION_FETCHER,
@@ -1212,7 +1209,7 @@ public class DataNodeTSIServiceImpl implements TSIEventHandler {
   }
 
   private void cleanupQueryExecution(Long queryId) {
-    IQueryExecution queryExecution = COORDINATOR.getQueryExecution(genQueryId(queryId));
+    IQueryExecution queryExecution = COORDINATOR.getQueryExecution(queryId);
     if (queryExecution != null) {
       queryExecution.stopAndCleanup();
     }
diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InternalServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InternalServiceImpl.java
index 4c7f08edf0..fb9480f272 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InternalServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InternalServiceImpl.java
@@ -50,6 +50,7 @@ import org.apache.iotdb.db.mpp.common.PlanFragmentId;
 import org.apache.iotdb.db.mpp.common.QueryId;
 import org.apache.iotdb.db.mpp.execution.fragment.FragmentInstanceInfo;
 import org.apache.iotdb.db.mpp.execution.fragment.FragmentInstanceManager;
+import org.apache.iotdb.db.mpp.plan.Coordinator;
 import org.apache.iotdb.db.mpp.plan.analyze.ClusterPartitionFetcher;
 import org.apache.iotdb.db.mpp.plan.analyze.QueryType;
 import org.apache.iotdb.db.mpp.plan.analyze.SchemaValidator;
@@ -59,7 +60,6 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.DeleteRegionNode;
 import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode;
 import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId;
 import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode;
-import org.apache.iotdb.db.query.control.SessionManager;
 import org.apache.iotdb.db.service.metrics.MetricsService;
 import org.apache.iotdb.db.service.metrics.enums.Metric;
 import org.apache.iotdb.db.service.metrics.enums.Tag;
@@ -338,8 +338,7 @@ public class InternalServiceImpl implements InternalService.Iface {
 
   @Override
   public TSStatus deleteRegion(TConsensusGroupId tconsensusGroupId) throws TException {
-    long queryIdRaw = SessionManager.getInstance().requestQueryId(false);
-    QueryId queryId = new QueryId(String.valueOf(queryIdRaw));
+    QueryId queryId = Coordinator.getInstance().createQueryId();
     PlanNodeId planNodeId = queryId.genPlanNodeId();
     DeleteRegionNode deleteRegionNode = new DeleteRegionNode(queryId.genPlanNodeId());
     ConsensusGroupId consensusGroupId =