You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2023/02/01 02:46:20 UTC
[iotdb] branch geely_car updated (84af179d1f -> fb6ff373d9)
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a change to branch geely_car
in repository https://gitbox.apache.org/repos/asf/iotdb.git
discard 84af179d1f fix the bug that timePartition cannot be found sometimes
add 8e35dc02ef [IOTDB-5320] Add executeAggregationQuery interface in session API
add e1ee85ba01 [IOTDB-5206] Set empty target resource to DELETED and then delete it after compaction (#8736)
add 648b047e18 [IOTDB-5381] Formatting variables output (#8821)
add 174242a678 [IOTDB-5418] Add metric for mods file (#8932)
add 58667c379b Fix English docs with Chinese characters (#8940)
add c807fca28b [IOTDB-5442] SchemaFile update StorageGroupNode asynchronously (#8939)
add b3117e4b32 [IOTDB-5395] Update file metrics when loading tsfile
add db6f17e77d Update SchemaFile to be compatible with MeasurementSchema.props (#8902)
add e93221f177 [IOTDB-5413] ReadWriteLock for CachedMTreeStore (#8864)
new fb6ff373d9 fix the bug that timePartition cannot be found sometimes
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (84af179d1f)
\
N -- N -- N refs/heads/geely_car (fb6ff373d9)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../iotdb/confignode/conf/ConfigNodeConfig.java | 13 +
.../iotdb/confignode/manager/node/NodeManager.java | 2 +-
.../iotdb/confignode/service/ConfigNode.java | 4 +
docs/UserGuide/API/Programming-Java-Native-API.md | 36 +-
docs/UserGuide/Monitor-Alert/Metric-Tool.md | 265 ++++++++----
docs/UserGuide/Query-Data/Overview.md | 52 ++-
.../UserGuide/QuickStart/Command-Line-Interface.md | 2 +-
docs/UserGuide/QuickStart/Files.md | 2 +-
docs/UserGuide/QuickStart/WayToGetIoTDB.md | 6 +-
.../UserGuide/API/Programming-Java-Native-API.md | 36 +-
docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 447 ++++++++++++---------
docs/zh/UserGuide/Query-Data/Overview.md | 52 ++-
.../main/java/org/apache/iotdb/SessionExample.java | 51 ++-
integration-test/import-control.xml | 3 +
.../db/it/aligned/IoTDBAlignedSeriesQueryIT.java | 10 -
.../apache/iotdb/db/it/utils/AlignedWriteUtil.java | 12 +
.../org/apache/iotdb/db/it/utils/TestUtils.java | 31 ++
.../iotdb/session/it/IoTDBSessionQueryIT.java | 327 +++++++++++++++
.../java/org/apache/iotdb/isession/ISession.java | 25 ++
.../apache/iotdb/isession/pool/ISessionPool.java | 27 +-
.../commons/schema/tree/AbstractTreeVisitor.java | 13 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 29 +-
.../iotdb/db/engine/TsFileMetricManager.java | 28 ++
.../exception/CompactionExceptionHandler.java | 11 +-
.../execute/recover/CompactionRecoverTask.java | 110 ++---
.../execute/task/CrossSpaceCompactionTask.java | 33 +-
.../execute/task/InnerSpaceCompactionTask.java | 19 +-
.../compaction/execute/utils/CompactionUtils.java | 21 +-
.../execute/utils/log/CompactionLogAnalyzer.java | 26 +-
.../execute/utils/log/CompactionLogger.java | 12 +
.../execute/utils/log/TsFileIdentifier.java | 6 +-
.../writer/AbstractCrossCompactionWriter.java | 5 +-
.../writer/AbstractInnerCompactionWriter.java | 4 +-
.../db/engine/modification/ModificationFile.java | 9 +
.../iotdb/db/engine/storagegroup/DataRegion.java | 28 ++
.../db/engine/storagegroup/TsFileManager.java | 4 +-
.../db/engine/storagegroup/TsFileResource.java | 7 +-
.../iotdb/db/metadata/mtree/IMTreeBelowSG.java | 2 +-
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 57 +--
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 25 +-
.../db/metadata/mtree/store/CachedMTreeStore.java | 193 ++++++---
.../iotdb/db/metadata/mtree/store/IMTreeStore.java | 2 +
.../db/metadata/mtree/store/MemMTreeStore.java | 5 +
.../store/ReentrantReadOnlyCachedMTreeStore.java | 130 ++++++
.../mtree/store/StampedWriterPreferredLock.java | 207 ++++++++++
.../mtree/store/disk/cache/CacheManager.java | 38 +-
.../mtree/store/disk/cache/ICacheManager.java | 3 +
.../mtree/store/disk/schemafile/RecordUtils.java | 23 +-
.../db/metadata/mtree/traverser/Traverser.java | 17 +-
.../mtree/traverser/basic/DatabaseTraverser.java | 6 +-
.../mtree/traverser/basic/EntityTraverser.java | 6 +-
.../mtree/traverser/basic/MNodeTraverser.java | 6 +-
.../traverser/basic/MeasurementTraverser.java | 6 +-
...atabaseCounter.java => MeasurementCounter.java} | 9 +-
.../mtree/traverser/updater/EntityUpdater.java | 6 +-
.../traverser/updater/MeasurementUpdater.java | 6 +-
.../metadata/rescon/SchemaStatisticsManager.java | 4 +-
.../schemaregion/SchemaRegionMemoryImpl.java | 12 +-
.../schemaregion/SchemaRegionSchemaFileImpl.java | 13 +-
.../db/mpp/aggregation/AccumulatorFactory.java | 8 +-
.../SlidingWindowAggregatorFactory.java | 4 +-
.../schematree/visitor/SchemaTreeVisitor.java | 1 +
.../db/mpp/plan/parser/StatementGenerator.java | 59 +++
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 10 +-
.../plan/parameter/AggregationDescriptor.java | 7 +-
.../planner/plan/parameter/AggregationType.java | 113 ------
.../statement/component/GroupByTimeComponent.java | 2 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 1 +
.../iotdb/db/service/metrics/FileMetrics.java | 18 +
.../service/thrift/impl/ClientRPCServiceImpl.java | 75 ++++
.../org/apache/iotdb/db/utils/SchemaUtils.java | 12 +-
.../FastCrossCompactionPerformerTest.java | 6 +-
.../ReadPointCompactionPerformerTest.java | 13 +-
.../cross/CrossSpaceCompactionExceptionTest.java | 200 +++++++++
.../RewriteCrossSpaceCompactionRecoverTest.java | 266 ++++++++++++
.../inner/InnerSpaceCompactionExceptionTest.java | 178 ++++++++
.../SizeTieredCompactionRecoverTest.java | 235 +++++++++++
.../mtree/lock/StampedWriterPreferredLockTest.java | 212 ++++++++++
.../metadata/mtree/schemafile/RecordUtilTests.java | 10 +-
.../metadata/mtree/schemafile/SchemaFileTest.java | 2 +-
.../iotdb/db/mpp/aggregation/AccumulatorTest.java | 22 +-
.../operator/AggregationOperatorTest.java | 48 +--
.../AlignedSeriesAggregationScanOperatorTest.java | 98 ++---
.../mpp/execution/operator/OperatorMemoryTest.java | 36 +-
.../operator/RawDataAggregationOperatorTest.java | 126 +++---
.../SeriesAggregationScanOperatorTest.java | 98 ++---
.../SlidingWindowAggregationOperatorTest.java | 36 +-
.../operator/VerticallyConcatOperatorTest.java | 6 +-
.../plan/analyze/AggregationDescriptorTest.java | 22 +-
.../iotdb/db/mpp/plan/plan/LogicalPlannerTest.java | 4 +-
.../db/mpp/plan/plan/QueryLogicalPlanUtil.java | 92 ++---
.../distribution/AggregationDistributionTest.java | 58 +--
.../node/process/AggregationNodeSerdeTest.java | 118 +++---
.../node/process/GroupByLevelNodeSerdeTest.java | 8 +-
.../plan/node/process/GroupByTagNodeSerdeTest.java | 10 +-
.../source/SeriesAggregationScanNodeSerdeTest.java | 4 +-
.../java/org/apache/iotdb/session/Session.java | 104 +++++
.../apache/iotdb/session/SessionConnection.java | 92 +++++
.../org/apache/iotdb/session/pool/SessionPool.java | 110 +++++
thrift-commons/src/main/thrift/common.thrift | 13 +
thrift/src/main/thrift/client.thrift | 25 +-
101 files changed, 3979 insertions(+), 1127 deletions(-)
create mode 100644 integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionQueryIT.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/ReentrantReadOnlyCachedMTreeStore.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/StampedWriterPreferredLock.java
copy server/src/main/java/org/apache/iotdb/db/metadata/mtree/traverser/counter/{DatabaseCounter.java => MeasurementCounter.java} (87%)
delete mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/parameter/AggregationType.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/metadata/mtree/lock/StampedWriterPreferredLockTest.java
[iotdb] 01/01: fix the bug that timePartition cannot be found sometimes
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch geely_car
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit fb6ff373d911015febdf006ee33255f0bb084ccd
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Tue Jan 31 23:17:42 2023 +0800
fix the bug that timePartition cannot be found sometimes
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../mpp/plan/analyze/ClusterPartitionFetcher.java | 45 ++++++++++++++++------
2 files changed, 35 insertions(+), 12 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 0f28f2fc89..5298dc16f9 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -875,7 +875,7 @@ public class IoTDBConfig {
* series partition
*/
private String seriesPartitionExecutorClass =
- "org.apache.iotdb.commons.partition.executor.hash.APHashExecutor";
+ "org.apache.iotdb.commons.partition.executor.hash.BKDRHashExecutor";
/** The number of series partitions in a database */
private int seriesPartitionSlotNum = 10000;
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterPartitionFetcher.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterPartitionFetcher.java
index 029c878114..117dc169ed 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterPartitionFetcher.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ClusterPartitionFetcher.java
@@ -58,9 +58,11 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
public class ClusterPartitionFetcher implements IPartitionFetcher {
@@ -260,9 +262,8 @@ public class ClusterPartitionFetcher implements IPartitionFetcher {
if (null == dataPartition) {
try (ConfigNodeClient client =
configNodeClientManager.borrowClient(ConfigNodeInfo.configNodeRegionId)) {
- TDataPartitionTableResp dataPartitionTableResp =
- client.getOrCreateDataPartitionTable(
- constructDataPartitionReq(splitDataPartitionQueryParams));
+ TDataPartitionReq req = constructDataPartitionReq(splitDataPartitionQueryParams);
+ TDataPartitionTableResp dataPartitionTableResp = client.getOrCreateDataPartitionTable(req);
if (dataPartitionTableResp.getStatus().getCode()
== TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
@@ -346,6 +347,22 @@ public class ClusterPartitionFetcher implements IPartitionFetcher {
}
}
+ private static class ComplexTimeSlotList {
+ Set<TTimePartitionSlot> timeSlotList;
+ boolean needLeftAll;
+ boolean needRightAll;
+
+ private ComplexTimeSlotList() {
+ timeSlotList = new HashSet<>();
+ needLeftAll = false;
+ needRightAll = false;
+ }
+
+ private void putTimeSlot(List<TTimePartitionSlot> slotList) {
+ timeSlotList.addAll(slotList);
+ }
+ }
+
private TDataPartitionReq constructDataPartitionReq(
Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap) {
Map<String, Map<TSeriesPartitionSlot, TTimeSlotList>> partitionSlotsMap = new HashMap<>();
@@ -353,16 +370,22 @@ public class ClusterPartitionFetcher implements IPartitionFetcher {
sgNameToQueryParamsMap.entrySet()) {
// for each sg
Map<TSeriesPartitionSlot, TTimeSlotList> deviceToTimePartitionMap = new HashMap<>();
+
+ Map<TSeriesPartitionSlot, ComplexTimeSlotList> seriesSlotTimePartitionMap = new HashMap<>();
+
for (DataPartitionQueryParam queryParam : entry.getValue()) {
- TTimeSlotList timePartitionSlotList =
- new TTimeSlotList(
- queryParam.getTimePartitionSlotList(),
- queryParam.isNeedLeftAll(),
- queryParam.isNeedRightAll());
- deviceToTimePartitionMap.put(
- partitionExecutor.getSeriesPartitionSlot(queryParam.getDevicePath()),
- timePartitionSlotList);
+ seriesSlotTimePartitionMap
+ .computeIfAbsent(
+ partitionExecutor.getSeriesPartitionSlot(queryParam.getDevicePath()),
+ k -> new ComplexTimeSlotList())
+ .putTimeSlot(queryParam.getTimePartitionSlotList());
}
+ seriesSlotTimePartitionMap.forEach(
+ (k, v) ->
+ deviceToTimePartitionMap.put(
+ k,
+ new TTimeSlotList(
+ new ArrayList<>(v.timeSlotList), v.needLeftAll, v.needRightAll)));
partitionSlotsMap.put(entry.getKey(), deviceToTimePartitionMap);
}
return new TDataPartitionReq(partitionSlotsMap);