You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2023/12/05 08:01:33 UTC
(iotdb) 01/03: merge master
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/time_recording
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7348120296795b43585721e70fc0f93ec6fb5ff1
Merge: 4bf62c08acf 0cc841b7639
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Fri Dec 1 09:39:40 2023 +0800
merge master
.github/workflows/grafana-plugin.yml | 2 +-
example/schema/pom.xml | 2 +-
.../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 25 ++-
.../db/it/auth/IoTDBTemplatePermissionIT.java | 5 +
.../org/apache/iotdb/db/it/utils/TestUtils.java | 12 +-
.../iotdb/pipe/PipeEnvironmentException.java | 12 +-
.../apache/iotdb/pipe/it/AbstractPipeDualIT.java | 60 ++++++
.../apache/iotdb/pipe/it/IoTDBPipeClusterIT.java | 65 +++----
.../pipe/it/IoTDBPipeConnectorParallelIT.java | 28 +--
.../apache/iotdb/pipe/it/IoTDBPipeDataSinkIT.java | 29 +--
.../apache/iotdb/pipe/it/IoTDBPipeExtractorIT.java | 71 +++----
.../apache/iotdb/pipe/it/IoTDBPipeLifeCycleIT.java | 29 +--
...orParallelIT.java => IoTDBPipeProcessorIT.java} | 68 +++----
.../apache/iotdb/pipe/it/IoTDBPipeProtocolIT.java | 27 ++-
.../iotdb/pipe/it/IoTDBPipeSingleEnvDemoIT.java | 16 +-
.../iotdb/pipe/it/IoTDBPipeSwitchStatusIT.java | 29 +--
.../apache/iotdb/pipe/it/IoTDBPipeSyntaxIT.java | 29 +--
.../java/org/apache/iotdb/pipe/api/access/Row.java | 10 +-
iotdb-connector/grafana-plugin/backend-compile.bat | 2 +-
iotdb-connector/grafana-plugin/backend-compile.sh | 31 ++-
iotdb-connector/grafana-plugin/go.mod | 35 ++--
.../iotdb/confignode/manager/ConfigManager.java | 2 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 19 +-
.../impl/pipe/task/DropPipeProcedureV2.java | 8 +-
.../impl/pipe/task/StartPipeProcedureV2.java | 20 +-
.../impl/pipe/task/StopPipeProcedureV2.java | 19 +-
.../consensus/iot/IoTConsensusServerImpl.java | 23 +--
.../consensus/iot/IoTConsensusServerMetrics.java | 2 +-
.../consensus/iot/client/DispatchLogHandler.java | 15 +-
.../iot/logdispatcher/IndexController.java | 2 -
.../consensus/iot/logdispatcher/LogDispatcher.java | 14 +-
.../service/IoTConsensusRPCServiceProcessor.java | 2 +-
.../apache/iotdb/consensus/ratis/utils/Utils.java | 19 +-
.../apache/iotdb/consensus/iot/ReplicateTest.java | 30 ++-
.../iotdb/consensus/ratis/RecoverReadTest.java | 8 +-
.../agent/plugin/PipeProcessorConstructor.java | 3 +
.../config/constant/PipeProcessorConstant.java | 12 ++
.../env/PipeTaskConnectorRuntimeEnvironment.java | 9 +-
.../env/PipeTaskExtractorRuntimeEnvironment.java | 9 +-
...va => PipeTaskProcessorRuntimeEnvironment.java} | 13 +-
.../plugin/env/PipeTaskRuntimeEnvironment.java | 8 +-
.../iotdb/db/pipe/event/common/row/PipeRow.java | 5 +
.../PipeHistoricalDataRegionTsFileExtractor.java | 59 +++---
.../downsampling/DownSamplingProcessor.java | 213 +++++++++++++++++++++
.../downsampling/PartialPathLastTimeCache.java | 111 +++++++++++
.../db/pipe/resource/memory/PipeMemoryBlock.java | 14 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 6 +-
.../common/schematree/ClusterSchemaTree.java | 8 +-
.../queryengine/common/schematree/ISchemaTree.java | 11 +-
.../db/queryengine/execution/driver/Driver.java | 4 -
.../execution/exchange/sink/LocalSinkChannel.java | 39 ++--
.../execution/exchange/sink/ShuffleSinkHandle.java | 27 ++-
.../execution/exchange/sink/SinkChannel.java | 73 ++++---
.../exchange/source/LocalSourceHandle.java | 15 +-
.../execution/exchange/source/SourceHandle.java | 20 +-
.../fragment/FragmentInstanceContext.java | 12 +-
.../fragment/FragmentInstanceExecution.java | 3 +-
.../execution/fragment/QueryContext.java | 4 +
.../queryengine/execution/memory/MemoryPool.java | 6 +-
.../queryengine/execution/operator/Operator.java | 12 +-
.../execution/operator/process/TopKOperator.java | 10 +
.../operator/schema/SchemaFetchScanOperator.java | 13 +-
.../source/AbstractDataSourceOperator.java | 5 +
.../operator/source/LastCacheScanOperator.java | 5 +
.../operator/source/ShowQueriesOperator.java | 5 +
.../schedule/DriverTaskTimeoutSentinelThread.java | 2 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 26 ++-
.../analyze/schema/AutoCreateSchemaExecutor.java | 61 ++++--
.../analyze/schema/ClusterSchemaFetchExecutor.java | 17 +-
.../plan/analyze/schema/ClusterSchemaFetcher.java | 13 +-
.../plan/analyze/schema/ISchemaFetcher.java | 6 +-
.../config/executor/ClusterConfigTaskExecutor.java | 3 +-
.../plan/planner/LogicalPlanBuilder.java | 6 +-
.../plan/planner/LogicalPlanVisitor.java | 3 +-
.../plan/planner/OperatorTreeGenerator.java | 3 +-
.../planner/distribution/ExchangeNodeAdder.java | 41 ++--
.../node/metedata/read/SchemaFetchScanNode.java | 17 +-
.../statement/internal/SchemaFetchStatement.java | 11 +-
.../template/AlterSchemaTemplateStatement.java | 19 --
.../binary/CompareBinaryColumnTransformer.java | 33 ++--
.../column/ternary/BetweenColumnTransformer.java | 21 +-
.../binary/CompareEqualToTransformer.java | 5 +-
.../binary/CompareGreaterEqualTransformer.java | 5 +-
.../binary/CompareGreaterThanTransformer.java | 5 +-
.../binary/CompareLessEqualTransformer.java | 5 +-
.../binary/CompareLessThanTransformer.java | 5 +-
.../binary/CompareNonEqualTransformer.java | 5 +-
.../transformer/ternary/BetweenTransformer.java | 15 +-
.../transformation/dag/util/TransformUtils.java | 41 ++--
.../schemaengine/schemaregion/ISchemaRegion.java | 5 +-
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 9 +-
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 9 +-
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 14 +-
.../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 14 +-
.../reader/impl/TimeseriesReaderWithViewFetch.java | 3 +-
.../iotdb/db/storageengine/buffer/ChunkCache.java | 10 +-
.../db/storageengine/dataregion/DataRegion.java | 2 -
.../io/LocalTextModificationAccessor.java | 3 -
.../read/reader/chunk/DiskAlignedChunkLoader.java | 6 +-
.../read/reader/chunk/DiskChunkLoader.java | 6 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 8 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 10 +-
.../schemaRegion/SchemaRegionBasicTest.java | 11 +-
.../schemaRegion/SchemaRegionTemplateTest.java | 9 +-
.../execution/operator/FillOperatorTest.java | 15 ++
.../operator/LastQueryMergeOperatorTest.java | 20 ++
.../execution/operator/LinearFillOperatorTest.java | 35 ++++
.../execution/operator/OperatorMemoryTest.java | 8 +-
.../schema/SchemaFetchScanOperatorTest.java | 4 +-
.../plan/analyze/ExpressionAnalyzerTest.java | 2 +-
.../plan/analyze/FakeSchemaFetcherImpl.java | 8 +-
.../queryengine/plan/plan/distribution/Util.java | 6 +-
.../queryengine/plan/plan/distribution/Util2.java | 6 +-
.../metadata/read/SchemaFetchMergeNodeTest.java | 1 +
.../metadata/read/SchemaFetchScanNodeTest.java | 2 +
.../db/storageengine/buffer/ChunkCacheTest.java | 6 +-
.../inner/InnerSpaceCompactionSelectorTest.java | 41 ++--
.../apache/iotdb/commons/path/PathPatternTree.java | 8 +
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 47 +++--
.../builtin/processor/DownSamplingProcessor.java} | 22 +--
.../builtin/processor/PlaceHolderProcessor.java | 68 +++++++
.../org/apache/iotdb/tsfile/utils/BytesUtils.java | 10 +
pom.xml | 4 +-
123 files changed, 1480 insertions(+), 824 deletions(-)
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
index 9af34817427,195986aae3f..0dc79dd5c48
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
@@@ -447,16 -445,6 +448,23 @@@ public class FragmentInstanceContext ex
unClosedFilePaths = null;
}
+ long endTime = System.currentTimeMillis();
+ releaseTime.set(endTime);
+ durationTime.set(endTime - executionStartTime.get());
+ LOGGER.warn(
- " ===== FragmentInstanceContext released, id: {}, releaseTime: {}, startTime: {}, durationTime:{}ms",
++ " ===== FragmentInstanceContext released, id: {}, "
++ + "releaseTime: {}, "
++ + "startTime: {}, "
++ + "durationTime: {}ms, "
++ + "loadTimeseriesMetadataDiskCount: {}, "
++ + "loadTimeseriesMetadataDiskTime: {}",
+ id,
+ releaseTime.get(),
+ executionStartTime.get(),
- durationTime.get());
++ durationTime.get(),
++ loadTimeSeriesMetadataDiskCount,
++ loadTimeSeriesMetadataDiskCount);
+
dataRegion = null;
globalTimeFilter = null;
sourcePaths = null;
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java
index b22f2b5f3cc,b22f2b5f3cc..2e2babe80ea
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java
@@@ -40,6 -40,6 +40,10 @@@ import java.util.Set
/** QueryContext contains the shared information with in a query. */
public class QueryContext {
++ public int loadTimeSeriesMetadataDiskCount = 0;
++ public long loadTimeSeriesMetadataDiskTime = 0;
++ public int loadTimeSeriesMetadataMemCount = 0;
++ public long loadTimeSeriesMetadataMemTime = 0;
/**
* The key is the path of a ModificationFile and the value is all Modifications in this file. We
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 0af2cd7a8c7,7a08d5b3e39..05fa39658a5
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@@ -190,11 -185,11 +190,19 @@@ public class FileLoaderUtils
}
return timeSeriesMetadata;
} finally {
++ long time = System.nanoTime() - t1;
++ if (loadFromMem) {
++ context.loadTimeSeriesMetadataMemCount += 1;
++ context.loadTimeSeriesMetadataMemTime += time;
++ } else {
++ context.loadTimeSeriesMetadataDiskCount += 1;
++ context.loadTimeSeriesMetadataDiskTime += time;
++ }
SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
loadFromMem
? LOAD_TIMESERIES_METADATA_NONALIGNED_MEM
: LOAD_TIMESERIES_METADATA_NONALIGNED_DISK,
-- System.nanoTime() - t1);
++ time);
}
}