You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by zy...@apache.org on 2023/03/09 02:20:21 UTC
[iotdb] branch rc/1.1.0 updated (06eee5eca1 -> c9008d7c49)
This is an automated email from the ASF dual-hosted git repository.
zyk pushed a change to branch rc/1.1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 06eee5eca1 [To rel/1.1] Fix memory calculation is not accurate in SystemInfo (#9238)
new a399982101 Set some runtime fields in FIConetxt to null when the FI is done
new c9008d7c49 [To rel/1.1][IOTDB-5240] Fix ConfigMTree snapshot deserialization while using template (#9248)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../iotdb/db/metadata/mtree/ConfigMTree.java | 37 ++++++++++++++--------
.../iotdb/db/mpp/execution/driver/DataDriver.java | 6 ++++
.../fragment/FragmentInstanceContext.java | 8 +++--
.../fragment/FragmentInstanceExecution.java | 4 +++
.../fragment/FragmentInstanceManager.java | 21 ++++++------
.../iotdb/db/metadata/mtree/ConfigMTreeTest.java | 3 ++
6 files changed, 52 insertions(+), 27 deletions(-)
[iotdb] 01/02: Set some runtime fields in FIConetxt to null when the FI is done
Posted by zy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zyk pushed a commit to branch rc/1.1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a3999821013c22186bbd0b74207200d12304b043
Author: Jackie Tien <ja...@gmail.com>
AuthorDate: Thu Mar 9 09:17:24 2023 +0800
Set some runtime fields in FIConetxt to null when the FI is done
---
.../iotdb/db/mpp/execution/driver/DataDriver.java | 6 ++++++
.../execution/fragment/FragmentInstanceContext.java | 8 ++++++--
.../fragment/FragmentInstanceExecution.java | 4 ++++
.../execution/fragment/FragmentInstanceManager.java | 21 ++++++++++-----------
4 files changed, 26 insertions(+), 13 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/DataDriver.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/DataDriver.java
index 287f034954..d7378a67b0 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/DataDriver.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/DataDriver.java
@@ -71,6 +71,12 @@ public class DataDriver extends Driver {
((DataDriverContext) driverContext).getSourceOperators();
if (sourceOperators != null && !sourceOperators.isEmpty()) {
QueryDataSource dataSource = initQueryDataSource();
+ if (dataSource == null) {
+ // if this driver is being initialized, meanwhile the whole FI was aborted or cancelled
+ // for some reasons, we may get null QueryDataSource here.
+ // And it's safe for us to throw this exception here in such case.
+ throw new IllegalStateException("QueryDataSource should never be null!");
+ }
sourceOperators.forEach(
sourceOperator -> {
// construct QueryDataSource for source operator
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceContext.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceContext.java
index 4d6c80233a..f810bc1ddb 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceContext.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceContext.java
@@ -55,7 +55,7 @@ public class FragmentInstanceContext extends QueryContext {
private IDataRegionForQuery dataRegion;
private Filter timeFilter;
- List<PartialPath> sourcePaths;
+ private List<PartialPath> sourcePaths;
// Shared by all scan operators in this fragment instance to avoid memory problem
private QueryDataSource sharedQueryDataSource;
/** closed tsfile used in this fragment instance */
@@ -354,7 +354,7 @@ public class FragmentInstanceContext extends QueryContext {
* All file paths used by this fragment instance must be cleared and thus the usage reference must
* be decreased.
*/
- protected void releaseResource() {
+ protected synchronized void releaseResource() {
for (TsFileResource tsFile : closedFilePaths) {
FileReaderManager.getInstance().decreaseFileReaderReference(tsFile, true);
}
@@ -363,5 +363,9 @@ public class FragmentInstanceContext extends QueryContext {
FileReaderManager.getInstance().decreaseFileReaderReference(tsFile, false);
}
unClosedFilePaths = null;
+ dataRegion = null;
+ timeFilter = null;
+ sourcePaths = null;
+ sharedQueryDataSource = null;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceExecution.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceExecution.java
index 2415bed270..f9c17c9963 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceExecution.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceExecution.java
@@ -99,6 +99,10 @@ public class FragmentInstanceExecution {
context.getFailureInfoList());
}
+ public long getStartTime() {
+ return context.getStartTime();
+ }
+
public FragmentInstanceStateMachine getStateMachine() {
return stateMachine;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceManager.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceManager.java
index 4c4dd4187b..6e5390fdc1 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceManager.java
@@ -92,12 +92,12 @@ public class FragmentInstanceManager {
this.infoCacheTime = new Duration(5, TimeUnit.MINUTES);
ScheduledExecutorUtil.safelyScheduleWithFixedDelay(
- instanceManagementExecutor, this::removeOldInstances, 200, 200, TimeUnit.MILLISECONDS);
+ instanceManagementExecutor, this::removeOldInstances, 2000, 2000, TimeUnit.MILLISECONDS);
ScheduledExecutorUtil.safelyScheduleWithFixedDelay(
instanceManagementExecutor,
this::cancelTimeoutFlushingInstances,
- 200,
- 200,
+ 2000,
+ 2000,
TimeUnit.MILLISECONDS);
this.intoOperationExecutor =
@@ -297,14 +297,13 @@ public class FragmentInstanceManager {
private void cancelTimeoutFlushingInstances() {
long now = System.currentTimeMillis();
- instanceContext.entrySet().stream()
- .filter(
- entry -> {
- FragmentInstanceContext context = entry.getValue();
- return context.getStateMachine().getState() == FragmentInstanceState.FLUSHING
- && (now - context.getStartTime()) > QUERY_TIMEOUT_MS;
- })
- .forEach(entry -> entry.getValue().failed(new TimeoutException()));
+ instanceExecution.forEach(
+ (key, execution) -> {
+ if (execution.getStateMachine().getState() == FragmentInstanceState.FLUSHING
+ && (now - execution.getStartTime()) > QUERY_TIMEOUT_MS) {
+ execution.getStateMachine().failed(new TimeoutException());
+ }
+ });
}
public ExecutorService getIntoOperationExecutor() {
[iotdb] 02/02: [To rel/1.1][IOTDB-5240] Fix ConfigMTree snapshot deserialization while using template (#9248)
Posted by zy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zyk pushed a commit to branch rc/1.1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c9008d7c49ff41ba37fcb2d7af74a05dd6f8a591
Author: Marcos_Zyk <38...@users.noreply.github.com>
AuthorDate: Thu Mar 9 10:04:21 2023 +0800
[To rel/1.1][IOTDB-5240] Fix ConfigMTree snapshot deserialization while using template (#9248)
---
.../iotdb/db/metadata/mtree/ConfigMTree.java | 37 ++++++++++++++--------
.../iotdb/db/metadata/mtree/ConfigMTreeTest.java | 3 ++
2 files changed, 26 insertions(+), 14 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/ConfigMTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/ConfigMTree.java
index 8c2649018c..1f08e42efb 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/ConfigMTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/ConfigMTree.java
@@ -636,19 +636,23 @@ public class ConfigMTree {
public void deserialize(InputStream inputStream) throws IOException {
byte type = ReadWriteIOUtils.readByte(inputStream);
- if (type != STORAGE_GROUP_MNODE_TYPE) {
- logger.error("Wrong node type. Cannot deserialize MTreeAboveSG from given buffer");
- return;
- }
- StorageGroupMNode storageGroupMNode = deserializeStorageGroupMNode(inputStream);
+ String name = null;
+ int childNum = 0;
+ Stack<Pair<InternalMNode, Boolean>> stack = new Stack<>();
+ StorageGroupMNode storageGroupMNode;
InternalMNode internalMNode;
- Stack<InternalMNode> stack = new Stack<>();
- stack.push(storageGroupMNode);
-
- String name = storageGroupMNode.getName();
- int childNum = 0;
+ if (type == STORAGE_GROUP_MNODE_TYPE) {
+ storageGroupMNode = deserializeStorageGroupMNode(inputStream);
+ name = storageGroupMNode.getName();
+ stack.push(new Pair<>(storageGroupMNode, true));
+ } else {
+ internalMNode = deserializeInternalMNode(inputStream);
+ childNum = ReadWriteIOUtils.readInt(inputStream);
+ name = internalMNode.getName();
+ stack.push(new Pair<>(internalMNode, false));
+ }
while (!PATH_ROOT.equals(name)) {
type = ReadWriteIOUtils.readByte(inputStream);
@@ -656,17 +660,22 @@ public class ConfigMTree {
case INTERNAL_MNODE_TYPE:
internalMNode = deserializeInternalMNode(inputStream);
childNum = ReadWriteIOUtils.readInt(inputStream);
+ boolean hasDB = false;
while (childNum > 0) {
- internalMNode.addChild(stack.pop());
+ hasDB = stack.peek().right;
+ internalMNode.addChild(stack.pop().left);
childNum--;
}
- stack.push(internalMNode);
+ stack.push(new Pair<>(internalMNode, hasDB));
name = internalMNode.getName();
break;
case STORAGE_GROUP_MNODE_TYPE:
storageGroupMNode = deserializeStorageGroupMNode(inputStream);
childNum = 0;
- stack.push(storageGroupMNode);
+ while (!stack.isEmpty() && !stack.peek().right) {
+ storageGroupMNode.addChild(stack.pop().left);
+ }
+ stack.push(new Pair<>(storageGroupMNode, true));
name = storageGroupMNode.getName();
break;
default:
@@ -674,7 +683,7 @@ public class ConfigMTree {
return;
}
}
- this.root = stack.peek();
+ this.root = stack.peek().left;
}
private InternalMNode deserializeInternalMNode(InputStream inputStream) throws IOException {
diff --git a/server/src/test/java/org/apache/iotdb/db/metadata/mtree/ConfigMTreeTest.java b/server/src/test/java/org/apache/iotdb/db/metadata/mtree/ConfigMTreeTest.java
index ee4a8a3689..53bb7e3432 100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/mtree/ConfigMTreeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/mtree/ConfigMTreeTest.java
@@ -261,6 +261,7 @@ public class ConfigMTreeTest {
storageGroupMNode.setDataReplicationFactor(i);
storageGroupMNode.setSchemaReplicationFactor(i);
storageGroupMNode.setTimePartitionInterval(i);
+ root.getNodeWithAutoCreate(pathList[i].concatNode("a")).setSchemaTemplateId(i);
}
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
@@ -277,6 +278,8 @@ public class ConfigMTreeTest {
Assert.assertEquals(i, storageGroupSchema.getSchemaReplicationFactor());
Assert.assertEquals(i, storageGroupSchema.getDataReplicationFactor());
Assert.assertEquals(i, storageGroupSchema.getTimePartitionInterval());
+ Assert.assertEquals(
+ i, newTree.getNodeWithAutoCreate(pathList[i].concatNode("a")).getSchemaTemplateId());
}
Assert.assertEquals(