You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2023/11/13 03:45:59 UTC
(iotdb) branch beyyes/multi_devices_fe updated (08e64e57f04 -> f25db9683c7)
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a change to branch beyyes/multi_devices_fe
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 08e64e57f04 fix compile error with master
add 93de59e4db0 Pipe: add connector's pending tsfile event count to help isTsFileEventCountInQueueExceededLimit judgement when hybrid mode is enabled (#11476)
add 1df8ec25eea Make all built-in scalar UDF into batch processing
add aa99a220bc3 Pipe: preliminary SSL support for pipe connector (sink) sync client (#11465)
add 3244255e940 fix empty resource recovered from wal (#11452)
add 4e686b9a824 Pipe: Enable github machines instead of self-hosted machines to run pipe IT (#11435)
add 256194a9e28 [IOTDB-6240] Pipe: Fix bug with PipeMemoryManager allocating memory for the DisruptorQueue (#11485)
add cc5ed977507 Pipe: refined key set of tags for pipe metrics & fixed hit rate in PipeWALInsertNodeCacheMetrics always `1.0`(#11484)
add fd3911f4ca6 Pipe: convert the value of source, processor, sink key to lowercase & convert `SHOW_PIPE_PLUGINS_BLACKLIST` keys to uppercase (#11487)
add 0f328ce5eb9 Fix empty tsfile genarated when all datatypes in one insert mismatching (#11456)
add f7b26846a0a Add Metric Prometheus Reporter IT (#11443)
add 0b8faac5f75 PBTree: Logging Disabled with Ratis and Pre-Allocation Disabled for devices using template (#11467)
add f594958f0da Feat/configurable home directory (#11492)
add ed3817eaac3 fix: Increase the timeout even more to finally make the build succeed.
add 9077f03e1a4 Fix NPE in SchemaStatisticsTest (#11490)
add 4b0fc1d7ec6 [IOTDB-6239] Show regions display error create time (#11481)
add b478d96ad21 Fix datanode stuck when meets disk error (#11486)
add 16eee23443b Simplify library-udf packaging result
add 058c5f6eab6 [IOTDB-6241] Load: Check System READ_ONLY mode in Analysis, Dispatch and Writing Stage (#11488)
add 3e1feac4d9e [IOTDB-1476]IoTDB support IPv6 (#11457)
add 8f3729fb825 Update ReadMe in IT module (#11499)
add 31d0c5bc1e8 Pipe: Added check in PipeSubtask submit logic to ensure one PipeConnectorSubtask instance is running in only one thread (#11474)
add a4510266fdb [IOTDB-6206] Metric Core: Shrink ringBuffer length to optimize histogram and timer's record performance (#11500)
add 1f518743f9f Fix aggregation error when inserting negative timestamp
add 05a608f8ea1 Pipe: Make each connector subtask inject cron event at fix rate to avoid random time of batch transmission (#11501)
add 1ea88fa2030 [IOTDB-6238] Fix delete timeSeries / deactivate template procedure rollback logic (#11479)
add 45c2fdf5535 Pipe: introduce pipe_max_allowed_pinned_memtable_count param for realtime hybrid mode (#11505)
add e094b3649f5 add time cost for datanode start (#11498)
add 33a3d73ed34 Pipe IT: adjust -DintegrationTest.forkCount & -DDataNodeMaxHeapSize in pipe-it-2cluster.yml to provide more stable env (#11504)
add bbc57ec98ff Check empty target file after sequence InnerSpaceCompactionTask (#11454)
add bb79cf16592 Add compaction logs and metrics (#11448)
add 98c4aa324ec Pipe: skip pipe CN heartbeat and sync when `PipeTaskCoordinatorLock` is held by another thread (#11509)
add d3548ed2acd [IOTDB-6209] Pipe: Solving the topological order of the progress index in the historical data collection phase (#11478)
add 8901076ec17 [IOTDB-6227] Add retry for RatisConsensus read (#11458)
add 58d3a129113 Fix exception add database name message (#11507)
add 32faf845ec3 add mod file exist judgement in QueryContext
add 9cd0fc401b2 spotless
add 6cc4bf80c57 perfect impl
new 2ee47ffa85c Merge branch 'beyyes/perfect_modfile' into beyyes/multi_devices_fe
new f25db9683c7 add template examination for TemplatedDeviceAnalyze
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/pipe-it-2cluster.yml | 9 +-
Jenkinsfile | 2 +-
distribution/src/assembly/library-udf.xml | 23 +-
integration-test/README.md | 24 +-
.../org/apache/iotdb/it/env/cluster/EnvUtils.java | 5 +-
.../it/env/cluster/config/MppConfigNodeConfig.java | 7 +
.../it/env/cluster/config/MppDataNodeConfig.java | 7 +
.../iotdb/it/env/cluster/env/AbstractEnv.java | 53 ++--
.../it/env/cluster/node/AbstractNodeWrapper.java | 7 +
.../it/env/cluster/node/ConfigNodeWrapper.java | 7 +-
.../iotdb/it/env/cluster/node/DataNodeWrapper.java | 5 +-
.../env/remote/config/RemoteConfigNodeConfig.java | 9 +-
.../it/env/remote/config/RemoteDataNodeConfig.java | 9 +-
.../iotdb/it/env/remote/env/RemoteServerEnv.java | 17 ++
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 32 +++
.../apache/iotdb/itbase/env/BaseNodeWrapper.java | 2 +
.../apache/iotdb/itbase/env/ConfigNodeConfig.java | 6 +-
.../apache/iotdb/itbase/env/DataNodeConfig.java | 6 +-
.../apache/iotdb/db/it/IoTDBInsertWithQueryIT.java | 23 ++
.../IoTDBMetricIT.java} | 32 ++-
.../main/java/org/apache/iotdb/jdbc/Config.java | 2 +
.../src/main/java/org/apache/iotdb/jdbc/Utils.java | 27 +-
.../test/java/org/apache/iotdb/jdbc/UtilsTest.java | 25 ++
.../main/java/org/apache/iotdb/rpc/UrlUtils.java | 52 ++++
.../java/org/apache/iotdb/rpc/UrlUtilsTest.java | 61 ++++
.../apache/iotdb/session/util/SessionUtils.java | 18 +-
iotdb-connector/grafana-plugin/go.mod | 2 +-
.../write/region/CreateRegionGroupsPlan.java | 54 ++--
.../manager/load/balancer/RegionBalancer.java | 2 +-
.../pipe/runtime/PipeHeartbeatScheduler.java | 6 +
.../manager/pipe/runtime/PipeMetaSyncer.java | 8 +-
.../manager/pipe/task/PipeTaskCoordinator.java | 7 +-
.../manager/pipe/task/PipeTaskCoordinatorLock.java | 4 +
.../partition/DatabasePartitionTable.java | 6 +-
.../persistence/partition/PartitionInfo.java | 6 +-
.../impl/schema/DeactivateTemplateProcedure.java | 40 +--
.../impl/schema/DeleteDatabaseProcedure.java | 6 +-
.../impl/schema/DeleteTimeSeriesProcedure.java | 44 +--
.../impl/schema/UnsetTemplateProcedure.java | 9 +-
.../statemachine/CreateRegionGroupsProcedure.java | 12 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 5 +-
.../confignode/persistence/PartitionInfoTest.java | 5 +
.../impl/CreateRegionGroupsProcedureTest.java | 4 +
.../ratis/ApplicationStateMachineProxy.java | 16 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 91 +++---
.../iotdb/consensus/ratis/utils/Retriable.java | 107 +++++++
.../iotdb/consensus/ratis/utils/RetryPolicy.java | 79 ++++++
.../apache/iotdb/consensus/ratis/utils/Utils.java | 10 +
.../iotdb/consensus/ratis/RecoverReadTest.java | 3 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 4 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +-
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 5 +
...geException.java => LoadReadOnlyException.java} | 9 +-
.../pipe/agent/runtime/PipeCronEventInjector.java | 4 +-
.../config/constant/PipeConnectorConstant.java | 4 +
.../protocol/legacy/IoTDBLegacyPipeConnector.java | 31 +-
.../thrift/async/IoTDBThriftAsyncConnector.java | 9 +-
.../thrift/sync/IoTDBThriftSyncConnector.java | 69 +++--
.../sync/IoTDBThriftSyncConnectorClient.java | 29 +-
.../event/common/heartbeat/PipeHeartbeatEvent.java | 7 +-
.../pipe/extractor/IoTDBDataRegionExtractor.java | 21 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 21 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 58 ++--
.../PipeRealtimeDataRegionLogExtractor.java | 2 +-
.../realtime/assigner/DisruptorQueue.java | 4 +-
.../iotdb/db/pipe/metric/PipeConnectorMetrics.java | 86 +++++-
.../iotdb/db/pipe/metric/PipeExtractorMetrics.java | 114 ++++++--
.../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 86 +++++-
.../db/pipe/resource/memory/PipeMemoryManager.java | 10 +-
.../db/pipe/task/stage/PipeTaskExtractorStage.java | 11 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 5 +
.../iotdb/db/pipe/task/subtask/PipeSubtask.java | 14 +-
.../subtask/connector/PipeConnectorSubtask.java | 121 ++++++--
.../connector/PipeConnectorSubtaskManager.java | 21 +-
.../subtask/processor/PipeProcessorSubtask.java | 21 +-
.../iotdb/db/protocol/client/ConfigNodeInfo.java | 14 +-
.../SingleTimeWindowIterator.java | 3 +-
.../execution/fragment/QueryContext.java | 16 +-
.../execution/operator/AggregationUtil.java | 2 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 31 +-
.../plan/analyze/LoadTsfileAnalyzer.java | 11 +
.../plan/analyze/TemplatedDeviceAnalyze.java | 62 +++-
.../db/queryengine/plan/parser/ASTVisitor.java | 19 +-
.../plan/planner/plan/node/write/InsertNode.java | 7 +
.../planner/plan/parameter/SeriesScanOptions.java | 7 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 9 +-
.../mtree/impl/pbtree/cache/CacheManager.java | 35 ++-
.../mtree/impl/pbtree/schemafile/SchemaFile.java | 35 ++-
.../schemafile/pagemgr/BTreePageManager.java | 6 +-
.../pbtree/schemafile/pagemgr/PageManager.java | 71 +++--
.../java/org/apache/iotdb/db/service/DataNode.java | 63 +++--
.../db/service/metrics/CompactionMetrics.java | 311 +++++++++++++++++++++
.../iotdb/db/storageengine/StorageEngine.java | 7 +
.../db/storageengine/dataregion/DataRegion.java | 48 +++-
.../impl/ReadChunkCompactionPerformer.java | 3 +
.../execute/task/AbstractCompactionTask.java | 63 ++++-
.../execute/task/CrossSpaceCompactionTask.java | 7 +-
.../execute/task/InnerSpaceCompactionTask.java | 35 +--
.../task/InsertionCrossSpaceCompactionTask.java | 20 +-
.../compaction/io/CompactionTsFileWriter.java | 11 +
.../schedule/CompactionScheduleSummary.java | 64 +++++
.../compaction/schedule/CompactionScheduler.java | 36 ++-
.../compaction/schedule/CompactionWorker.java | 13 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 11 +-
.../dataregion/flush/MemTableFlushTask.java | 4 +-
.../dataregion/tsfile/TsFileResource.java | 4 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 1 +
.../wal/utils/listener/AbstractResultListener.java | 2 +
.../db/storageengine/rescon/disk/TierManager.java | 3 +
.../db/storageengine/rescon/memory/SystemInfo.java | 52 +++-
.../org/apache/iotdb/db/utils/DateTimeUtils.java | 14 -
.../mtree/schemafile/SchemaFileLogTest.java | 13 +
.../schemaRegion/SchemaRegionTemplateTest.java | 42 +--
.../PipeConnectorSubtaskExecutorTest.java | 3 +
.../PipeProcessorSubtaskExecutorTest.java | 3 +
.../storageengine/dataregion/DataRegionTest.java | 120 ++++++++
.../TsFileResourceProgressIndexTest.java | 162 ++++++++++-
.../FastInnerCompactionPerformerTest.java | 2 +-
.../compaction/ReadChunkInnerCompactionTest.java | 80 ++++++
...sSpaceCompactionWithReadPointPerformerTest.java | 4 +-
.../inner/InnerCompactionEmptyTsFileTest.java | 2 +-
.../inner/InnerCompactionSchedulerTest.java | 10 +-
.../iotdb/metrics/core/IoTDBMetricManager.java | 3 +-
.../iotdb/metrics/AbstractMetricManager.java | 10 +-
.../iotdb/metrics/impl/DoNothingMetricManager.java | 2 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 27 ++
.../iotdb/commons/conf/CommonDescriptor.java | 10 +
.../apache/iotdb/commons/conf/IoTDBConstant.java | 5 +
.../commons/consensus/index/ProgressIndex.java | 146 ++++++++--
.../consensus/index/impl/HybridProgressIndex.java | 36 ++-
.../consensus/index/impl/IoTProgressIndex.java | 19 +-
.../consensus/index/impl/MinimumProgressIndex.java | 12 +-
.../consensus/index/impl/RecoverProgressIndex.java | 22 +-
.../consensus/index/impl/SimpleProgressIndex.java | 11 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 12 +
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 31 +-
.../iotdb/commons/service/RegisterManager.java | 6 +
.../iotdb/commons/service/metric/enums/Metric.java | 5 +
.../iotdb/commons/service/metric/enums/Tag.java | 4 +-
.../commons/udf/builtin/String/UDTFConcat.java | 63 ++++-
.../commons/udf/builtin/String/UDTFEndsWith.java | 20 ++
.../commons/udf/builtin/String/UDTFLower.java | 21 ++
.../commons/udf/builtin/String/UDTFStartsWith.java | 36 ++-
.../commons/udf/builtin/String/UDTFStrCompare.java | 40 ++-
.../commons/udf/builtin/String/UDTFStrLength.java | 36 ++-
.../commons/udf/builtin/String/UDTFStrLocate.java | 37 ++-
.../iotdb/commons/udf/builtin/String/UDTFTrim.java | 36 ++-
.../commons/udf/builtin/String/UDTFUpper.java | 21 ++
.../apache/iotdb/commons/udf/builtin/UDTFAbs.java | 85 ++++++
.../iotdb/commons/udf/builtin/UDTFConst.java | 127 ++++++++-
.../iotdb/commons/udf/builtin/UDTFConstE.java | 34 ++-
.../iotdb/commons/udf/builtin/UDTFConstPi.java | 34 ++-
.../iotdb/commons/udf/builtin/UDTFContains.java | 36 ++-
.../iotdb/commons/udf/builtin/UDTFInRange.java | 121 +++++++-
.../iotdb/commons/udf/builtin/UDTFMatches.java | 35 ++-
.../apache/iotdb/commons/udf/builtin/UDTFMath.java | 85 ++++++
.../iotdb/commons/udf/builtin/UDTFOnOff.java | 85 ++++++
.../iotdb/commons/utils/CommonDateTimeUtils.java | 15 +
.../apache/iotdb/commons/utils/NodeUrlUtils.java | 23 +-
.../iotdb/commons/pipe/PipeMetaDeSerTest.java | 4 +-
.../iotdb/commons/utils/NodeUrlUtilsTest.java | 33 +++
.../tsfile/write/chunk/AlignedChunkWriterImpl.java | 5 +
.../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 5 +
.../iotdb/tsfile/write/chunk/IChunkWriter.java | 3 +
library-udf/pom.xml | 5 +
.../apache/iotdb/library/anomaly/UDTFRange.java | 9 +-
.../apache/iotdb/library/dprofile/UDTFSample.java | 8 +-
.../iotdb/library/string/UDTFRegexMatch.java | 44 ++-
.../iotdb/library/string/UDTFRegexReplace.java | 46 ++-
.../iotdb/library/string/UDTFRegexSplit.java | 64 ++++-
.../iotdb/library/string/UDTFStrReplace.java | 45 ++-
171 files changed, 4218 insertions(+), 718 deletions(-)
copy integration-test/src/test/java/org/apache/iotdb/db/it/{selectinto/IoTDBSelectInto2IT.java => metric/IoTDBMetricIT.java} (65%)
create mode 100644 iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/UrlUtils.java
create mode 100644 iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/UrlUtilsTest.java
create mode 100644 iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/utils/Retriable.java
create mode 100644 iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/utils/RetryPolicy.java
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/{MergeException.java => LoadReadOnlyException.java} (77%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleSummary.java
(iotdb) 02/02: add template examination for TemplatedDeviceAnalyze
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/multi_devices_fe
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f25db9683c733c02435babd78459bf0f54828c90
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Mon Nov 13 11:40:17 2023 +0800
add template examination for TemplatedDeviceAnalyze
---
.../queryengine/plan/analyze/AnalyzeVisitor.java | 31 ++++++++---
.../plan/analyze/TemplatedDeviceAnalyze.java | 62 ++++++++++++++++++++--
2 files changed, 84 insertions(+), 9 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
index 6e9c298d3f9..70cd42448ba 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
@@ -266,12 +266,31 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
List<Pair<Expression, String>> outputExpressions;
if (queryStatement.isAlignByDevice()) {
+ // examine that if all devices are in same template
+ if (!queryStatement.isAggregationQuery() && !queryStatement.isSelectInto()) {
+ Template template = null;
+ List<PartialPath> devicePatternList = queryStatement.getFromComponent().getPrefixPaths();
+ for (PartialPath devicePath : devicePatternList) {
+ Map<Integer, Template> templateMap = schemaFetcher.checkAllRelatedTemplate(devicePath);
+ if (templateMap.size() == 1) {
+ if (template == null) {
+ template = templateMap.values().iterator().next();
+ } else {
+ if (templateMap.values().iterator().next().getId() != template.getId()) {
+ template = null;
+ break;
+ }
+ }
+ } else {
+ break;
+ }
+ }
- // TODO examine that if all devices are in same template
- if (!queryStatement.isAggregationQuery()) {
- return new TemplatedDeviceAnalyze(
- analysis, queryStatement, context, schemaTree, partitionFetcher)
- .visitQuery();
+ if (template != null) {
+ return new TemplatedDeviceAnalyze(
+ analysis, queryStatement, context, schemaTree, partitionFetcher)
+ .visitQuery();
+ }
}
List<PartialPath> deviceList = analyzeFrom(queryStatement, schemaTree);
@@ -1813,7 +1832,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
analysis.setGlobalTimeFilter(globalTimeFilter);
}
- private void analyzeFill(Analysis analysis, QueryStatement queryStatement) {
+ static void analyzeFill(Analysis analysis, QueryStatement queryStatement) {
if (queryStatement.getFillComponent() == null) {
return;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedDeviceAnalyze.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedDeviceAnalyze.java
index 6cef714dc6d..a1c0b2cb637 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedDeviceAnalyze.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedDeviceAnalyze.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.db.queryengine.plan.expression.Expression;
import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand;
import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression;
import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering;
+import org.apache.iotdb.db.queryengine.plan.statement.component.SortItem;
import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager;
import org.apache.iotdb.db.schemaengine.template.Template;
@@ -67,9 +68,11 @@ import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.END_TI
import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.WHERE_WRONG_TYPE_ERROR_MSG;
import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.analyzeDeviceViewSpecialProcess;
import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.analyzeExpressionType;
+import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.analyzeFill;
import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.analyzeOutput;
import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.checkDeviceViewInputUniqueness;
import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.getTimePartitionSlotList;
+import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.concatDeviceAndBindSchemaForExpression;
import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.getMeasurementExpression;
import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.normalizeExpression;
import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.searchAggregationExpressions;
@@ -133,7 +136,7 @@ public class TemplatedDeviceAnalyze {
}
}
- /** 并行 最上层比如加sort fe 线程管理 */
+ /** add multi-threading impl */
public Analysis visitQuery() {
long startTime = System.currentTimeMillis();
@@ -165,7 +168,7 @@ public class TemplatedDeviceAnalyze {
analysis.setDeviceList(deviceList);
// analyzeDeviceToGroupBy(analysis, queryStatement, schemaTree, deviceList);
- // analyzeDeviceToOrderBy(analysis, queryStatement, schemaTree, deviceList);
+ analyzeDeviceToOrderBy(analysis, queryStatement, schemaTree, deviceList);
// analyzeHaving(analysis, queryStatement, schemaTree, deviceList);
// analyzeDeviceToAggregation(analysis, queryStatement);
analyzeDeviceToSourceTransform(analysis, queryStatement);
@@ -184,7 +187,7 @@ public class TemplatedDeviceAnalyze {
// analyzeInto(analysis, queryStatement, deviceList, outputExpressions);
// analyzeGroupByTime(analysis, queryStatement);
- // analyzeFill(analysis, queryStatement);
+ analyzeFill(analysis, queryStatement);
// generate result set header according to output expressions
analyzeOutput(analysis, queryStatement, outputExpressions);
@@ -319,6 +322,59 @@ public class TemplatedDeviceAnalyze {
return outputExpressions;
}
+ private void analyzeDeviceToOrderBy(
+ Analysis analysis,
+ QueryStatement queryStatement,
+ ISchemaTree schemaTree,
+ List<PartialPath> deviceSet) {
+ if (!queryStatement.hasOrderByExpression()) {
+ return;
+ }
+
+ Map<String, Set<Expression>> deviceToOrderByExpressions = new LinkedHashMap<>();
+ Map<String, List<SortItem>> deviceToSortItems = new LinkedHashMap<>();
+ // build the device-view outputColumn for the sortNode above the deviceViewNode
+ Set<Expression> deviceViewOrderByExpression = new LinkedHashSet<>();
+ for (PartialPath device : deviceSet) {
+ Set<Expression> orderByExpressionsForOneDevice = new LinkedHashSet<>();
+ for (Expression expressionForItem : queryStatement.getExpressionSortItemList()) {
+ List<Expression> expressions =
+ concatDeviceAndBindSchemaForExpression(expressionForItem, device, schemaTree);
+ if (expressions.isEmpty()) {
+ throw new SemanticException(
+ String.format(
+ "%s in order by clause doesn't exist.", expressionForItem.getExpressionString()));
+ }
+ if (expressions.size() > 1) {
+ throw new SemanticException(
+ String.format(
+ "%s in order by clause shouldn't refer to more than one timeseries.",
+ expressionForItem.getExpressionString()));
+ }
+ expressionForItem = expressions.get(0);
+ TSDataType dataType = analyzeExpressionType(analysis, expressionForItem);
+ if (!dataType.isComparable()) {
+ throw new SemanticException(
+ String.format("The data type of %s is not comparable", dataType));
+ }
+
+ Expression deviceViewExpression = getMeasurementExpression(expressionForItem, analysis);
+ analyzeExpressionType(analysis, deviceViewExpression);
+
+ deviceViewOrderByExpression.add(deviceViewExpression);
+ orderByExpressionsForOneDevice.add(expressionForItem);
+ }
+ deviceToSortItems.put(
+ device.getFullPath(), queryStatement.getUpdatedSortItems(orderByExpressionsForOneDevice));
+ deviceToOrderByExpressions.put(device.getFullPath(), orderByExpressionsForOneDevice);
+ }
+
+ analysis.setOrderByExpressions(deviceViewOrderByExpression);
+ queryStatement.updateSortItems(deviceViewOrderByExpression);
+ analysis.setDeviceToSortItems(deviceToSortItems);
+ analysis.setDeviceToOrderByExpressions(deviceToOrderByExpressions);
+ }
+
private void analyzeDeviceToSourceTransform(Analysis analysis, QueryStatement queryStatement) {
if (queryStatement.isAggregationQuery()) {
Map<String, Set<Expression>> deviceToSourceTransformExpressions =
(iotdb) 01/02: Merge branch 'beyyes/perfect_modfile' into beyyes/multi_devices_fe
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/multi_devices_fe
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2ee47ffa85c1bdf80a7c68b5e17ed30db4ec746c
Merge: 08e64e57f04 6cc4bf80c57
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Fri Nov 10 16:57:30 2023 +0800
Merge branch 'beyyes/perfect_modfile' into beyyes/multi_devices_fe
.github/workflows/pipe-it-2cluster.yml | 9 +-
Jenkinsfile | 2 +-
distribution/src/assembly/library-udf.xml | 23 +-
integration-test/README.md | 24 +-
.../org/apache/iotdb/it/env/cluster/EnvUtils.java | 5 +-
.../it/env/cluster/config/MppConfigNodeConfig.java | 7 +
.../it/env/cluster/config/MppDataNodeConfig.java | 7 +
.../iotdb/it/env/cluster/env/AbstractEnv.java | 53 ++--
.../it/env/cluster/node/AbstractNodeWrapper.java | 7 +
.../it/env/cluster/node/ConfigNodeWrapper.java | 7 +-
.../iotdb/it/env/cluster/node/DataNodeWrapper.java | 5 +-
.../env/remote/config/RemoteConfigNodeConfig.java | 9 +-
.../it/env/remote/config/RemoteDataNodeConfig.java | 9 +-
.../iotdb/it/env/remote/env/RemoteServerEnv.java | 17 ++
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 32 +++
.../apache/iotdb/itbase/env/BaseNodeWrapper.java | 2 +
.../apache/iotdb/itbase/env/ConfigNodeConfig.java | 6 +-
.../apache/iotdb/itbase/env/DataNodeConfig.java | 6 +-
.../apache/iotdb/db/it/IoTDBInsertWithQueryIT.java | 23 ++
.../apache/iotdb/db/it/metric/IoTDBMetricIT.java | 68 +++++
.../main/java/org/apache/iotdb/jdbc/Config.java | 2 +
.../src/main/java/org/apache/iotdb/jdbc/Utils.java | 27 +-
.../test/java/org/apache/iotdb/jdbc/UtilsTest.java | 25 ++
.../main/java/org/apache/iotdb/rpc/UrlUtils.java | 52 ++++
.../java/org/apache/iotdb/rpc/UrlUtilsTest.java | 61 ++++
.../apache/iotdb/session/util/SessionUtils.java | 18 +-
iotdb-connector/grafana-plugin/go.mod | 2 +-
.../write/region/CreateRegionGroupsPlan.java | 54 ++--
.../manager/load/balancer/RegionBalancer.java | 2 +-
.../pipe/runtime/PipeHeartbeatScheduler.java | 6 +
.../manager/pipe/runtime/PipeMetaSyncer.java | 8 +-
.../manager/pipe/task/PipeTaskCoordinator.java | 7 +-
.../manager/pipe/task/PipeTaskCoordinatorLock.java | 4 +
.../partition/DatabasePartitionTable.java | 6 +-
.../persistence/partition/PartitionInfo.java | 6 +-
.../impl/schema/DeactivateTemplateProcedure.java | 40 +--
.../impl/schema/DeleteDatabaseProcedure.java | 6 +-
.../impl/schema/DeleteTimeSeriesProcedure.java | 44 +--
.../impl/schema/UnsetTemplateProcedure.java | 9 +-
.../statemachine/CreateRegionGroupsProcedure.java | 12 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 5 +-
.../confignode/persistence/PartitionInfoTest.java | 5 +
.../impl/CreateRegionGroupsProcedureTest.java | 4 +
.../ratis/ApplicationStateMachineProxy.java | 16 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 91 +++---
.../iotdb/consensus/ratis/utils/Retriable.java | 107 +++++++
.../iotdb/consensus/ratis/utils/RetryPolicy.java | 79 ++++++
.../apache/iotdb/consensus/ratis/utils/Utils.java | 10 +
.../iotdb/consensus/ratis/RecoverReadTest.java | 3 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 4 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +-
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 5 +
.../iotdb/db/exception/LoadReadOnlyException.java} | 26 +-
.../pipe/agent/runtime/PipeCronEventInjector.java | 4 +-
.../config/constant/PipeConnectorConstant.java | 4 +
.../protocol/legacy/IoTDBLegacyPipeConnector.java | 31 +-
.../thrift/async/IoTDBThriftAsyncConnector.java | 9 +-
.../thrift/sync/IoTDBThriftSyncConnector.java | 69 +++--
.../sync/IoTDBThriftSyncConnectorClient.java | 29 +-
.../event/common/heartbeat/PipeHeartbeatEvent.java | 7 +-
.../pipe/extractor/IoTDBDataRegionExtractor.java | 21 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 21 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 58 ++--
.../PipeRealtimeDataRegionLogExtractor.java | 2 +-
.../realtime/assigner/DisruptorQueue.java | 4 +-
.../iotdb/db/pipe/metric/PipeConnectorMetrics.java | 86 +++++-
.../iotdb/db/pipe/metric/PipeExtractorMetrics.java | 114 ++++++--
.../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 86 +++++-
.../db/pipe/resource/memory/PipeMemoryManager.java | 10 +-
.../db/pipe/task/stage/PipeTaskExtractorStage.java | 11 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 5 +
.../iotdb/db/pipe/task/subtask/PipeSubtask.java | 14 +-
.../subtask/connector/PipeConnectorSubtask.java | 121 ++++++--
.../connector/PipeConnectorSubtaskManager.java | 21 +-
.../subtask/processor/PipeProcessorSubtask.java | 21 +-
.../iotdb/db/protocol/client/ConfigNodeInfo.java | 14 +-
.../SingleTimeWindowIterator.java | 3 +-
.../execution/fragment/QueryContext.java | 16 +-
.../execution/operator/AggregationUtil.java | 2 +-
.../plan/analyze/LoadTsfileAnalyzer.java | 11 +
.../db/queryengine/plan/parser/ASTVisitor.java | 19 +-
.../plan/planner/plan/node/write/InsertNode.java | 7 +
.../planner/plan/parameter/SeriesScanOptions.java | 7 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 9 +-
.../mtree/impl/pbtree/cache/CacheManager.java | 35 ++-
.../mtree/impl/pbtree/schemafile/SchemaFile.java | 35 ++-
.../schemafile/pagemgr/BTreePageManager.java | 6 +-
.../pbtree/schemafile/pagemgr/PageManager.java | 71 +++--
.../java/org/apache/iotdb/db/service/DataNode.java | 63 +++--
.../db/service/metrics/CompactionMetrics.java | 311 +++++++++++++++++++++
.../iotdb/db/storageengine/StorageEngine.java | 7 +
.../db/storageengine/dataregion/DataRegion.java | 48 +++-
.../impl/ReadChunkCompactionPerformer.java | 3 +
.../execute/task/AbstractCompactionTask.java | 63 ++++-
.../execute/task/CrossSpaceCompactionTask.java | 7 +-
.../execute/task/InnerSpaceCompactionTask.java | 35 +--
.../task/InsertionCrossSpaceCompactionTask.java | 20 +-
.../compaction/io/CompactionTsFileWriter.java | 11 +
.../schedule/CompactionScheduleSummary.java | 64 +++++
.../compaction/schedule/CompactionScheduler.java | 36 ++-
.../compaction/schedule/CompactionWorker.java | 13 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 11 +-
.../dataregion/flush/MemTableFlushTask.java | 4 +-
.../dataregion/tsfile/TsFileResource.java | 4 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 1 +
.../wal/utils/listener/AbstractResultListener.java | 2 +
.../db/storageengine/rescon/disk/TierManager.java | 3 +
.../db/storageengine/rescon/memory/SystemInfo.java | 52 +++-
.../org/apache/iotdb/db/utils/DateTimeUtils.java | 14 -
.../mtree/schemafile/SchemaFileLogTest.java | 13 +
.../schemaRegion/SchemaRegionTemplateTest.java | 42 +--
.../PipeConnectorSubtaskExecutorTest.java | 3 +
.../PipeProcessorSubtaskExecutorTest.java | 3 +
.../storageengine/dataregion/DataRegionTest.java | 120 ++++++++
.../TsFileResourceProgressIndexTest.java | 162 ++++++++++-
.../FastInnerCompactionPerformerTest.java | 2 +-
.../compaction/ReadChunkInnerCompactionTest.java | 80 ++++++
...sSpaceCompactionWithReadPointPerformerTest.java | 4 +-
.../inner/InnerCompactionEmptyTsFileTest.java | 2 +-
.../inner/InnerCompactionSchedulerTest.java | 10 +-
.../iotdb/metrics/core/IoTDBMetricManager.java | 3 +-
.../iotdb/metrics/AbstractMetricManager.java | 10 +-
.../iotdb/metrics/impl/DoNothingMetricManager.java | 2 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 27 ++
.../iotdb/commons/conf/CommonDescriptor.java | 10 +
.../apache/iotdb/commons/conf/IoTDBConstant.java | 5 +
.../commons/consensus/index/ProgressIndex.java | 146 ++++++++--
.../consensus/index/impl/HybridProgressIndex.java | 36 ++-
.../consensus/index/impl/IoTProgressIndex.java | 19 +-
.../consensus/index/impl/MinimumProgressIndex.java | 12 +-
.../consensus/index/impl/RecoverProgressIndex.java | 22 +-
.../consensus/index/impl/SimpleProgressIndex.java | 11 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 12 +
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 31 +-
.../iotdb/commons/service/RegisterManager.java | 6 +
.../iotdb/commons/service/metric/enums/Metric.java | 5 +
.../iotdb/commons/service/metric/enums/Tag.java | 4 +-
.../commons/udf/builtin/String/UDTFConcat.java | 63 ++++-
.../commons/udf/builtin/String/UDTFEndsWith.java | 20 ++
.../commons/udf/builtin/String/UDTFLower.java | 21 ++
.../commons/udf/builtin/String/UDTFStartsWith.java | 36 ++-
.../commons/udf/builtin/String/UDTFStrCompare.java | 40 ++-
.../commons/udf/builtin/String/UDTFStrLength.java | 36 ++-
.../commons/udf/builtin/String/UDTFStrLocate.java | 37 ++-
.../iotdb/commons/udf/builtin/String/UDTFTrim.java | 36 ++-
.../commons/udf/builtin/String/UDTFUpper.java | 21 ++
.../apache/iotdb/commons/udf/builtin/UDTFAbs.java | 85 ++++++
.../iotdb/commons/udf/builtin/UDTFConst.java | 127 ++++++++-
.../iotdb/commons/udf/builtin/UDTFConstE.java | 34 ++-
.../iotdb/commons/udf/builtin/UDTFConstPi.java | 34 ++-
.../iotdb/commons/udf/builtin/UDTFContains.java | 36 ++-
.../iotdb/commons/udf/builtin/UDTFInRange.java | 121 +++++++-
.../iotdb/commons/udf/builtin/UDTFMatches.java | 35 ++-
.../apache/iotdb/commons/udf/builtin/UDTFMath.java | 85 ++++++
.../iotdb/commons/udf/builtin/UDTFOnOff.java | 85 ++++++
.../iotdb/commons/utils/CommonDateTimeUtils.java | 15 +
.../apache/iotdb/commons/utils/NodeUrlUtils.java | 23 +-
.../iotdb/commons/pipe/PipeMetaDeSerTest.java | 4 +-
.../iotdb/commons/utils/NodeUrlUtilsTest.java | 33 +++
.../tsfile/write/chunk/AlignedChunkWriterImpl.java | 5 +
.../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 5 +
.../iotdb/tsfile/write/chunk/IChunkWriter.java | 3 +
library-udf/pom.xml | 5 +
.../apache/iotdb/library/anomaly/UDTFRange.java | 9 +-
.../apache/iotdb/library/dprofile/UDTFSample.java | 8 +-
.../iotdb/library/string/UDTFRegexMatch.java | 44 ++-
.../iotdb/library/string/UDTFRegexReplace.java | 46 ++-
.../iotdb/library/string/UDTFRegexSplit.java | 64 ++++-
.../iotdb/library/string/UDTFStrReplace.java | 45 ++-
169 files changed, 4182 insertions(+), 714 deletions(-)