You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2024/10/21 09:59:49 UTC
(doris) branch branch-3.0 updated: [Feat](Nereids) support plan_replayer #36716 #41664 (#41933)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 88e4ccbd897 [Feat](Nereids) support plan_replayer #36716 #41664 (#41933)
88e4ccbd897 is described below
commit 88e4ccbd8972bad6e46235a92c672349185d8d28
Author: LiBinfeng <46...@users.noreply.github.com>
AuthorDate: Mon Oct 21 17:59:42 2024 +0800
[Feat](Nereids) support plan_replayer #36716 #41664 (#41933)
cherry-pick from master #36716 #41664
---
.../main/java/org/apache/doris/common/Config.java | 3 +
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 3 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 11 +-
.../java/org/apache/doris/analysis/StmtType.java | 1 +
.../apache/doris/catalog/ColocateGroupSchema.java | 5 +
.../apache/doris/catalog/ColocateTableIndex.java | 7 +
.../main/java/org/apache/doris/catalog/Table.java | 1 +
.../main/java/org/apache/doris/catalog/View.java | 4 +
.../doris/common/proc/FrontendsProcNode.java | 9 +
.../apache/doris/httpv2/rest/MinidumpAction.java | 71 ++++++++
.../org/apache/doris/nereids/CascadesContext.java | 31 +---
.../org/apache/doris/nereids/NereidsPlanner.java | 2 +-
.../org/apache/doris/nereids/cost/CostModelV1.java | 6 +-
.../nereids/jobs/cascades/DeriveStatsJob.java | 3 +-
.../apache/doris/nereids/minidump/Minidump.java | 27 ++-
.../doris/nereids/minidump/MinidumpUtils.java | 198 +++++++++++++++++----
.../doris/nereids/parser/LogicalPlanBuilder.java | 15 ++
.../doris/nereids/rules/analysis/BindRelation.java | 18 +-
.../apache/doris/nereids/trees/plans/PlanType.java | 3 +-
.../trees/plans/commands/ReplayCommand.java | 119 +++++++++++++
.../trees/plans/logical/LogicalOlapScan.java | 1 +
.../trees/plans/physical/AbstractPhysicalJoin.java | 2 +-
.../trees/plans/visitor/CommandVisitor.java | 5 +
.../org/apache/doris/persist/gson/GsonUtils.java | 5 +-
.../java/org/apache/doris/qe/ConnectContext.java | 24 ++-
.../java/org/apache/doris/qe/SessionVariable.java | 24 ---
.../java/org/apache/doris/qe/StmtExecutor.java | 22 +++
.../apache/doris/statistics/ColumnStatistic.java | 30 +++-
.../doris/nereids/minidump/MinidumpUtTest.java | 4 +-
.../doris/nereids/parser/NereidsParserTest.java | 15 ++
.../apache/doris/nereids/util/ReadLockTest.java | 44 +++--
.../eliminate_outer_join/eliminate_outer_join.out | 18 +-
.../suites/nereids_p0/hint/test_leading.groovy | 18 +-
33 files changed, 594 insertions(+), 155 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 62a6ebf1655..9811eef28b6 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2802,6 +2802,9 @@ public class Config extends ConfigBase {
@ConfField
public static String spilled_profile_storage_path = System.getenv("LOG_DIR") + File.separator + "profile";
+ @ConfField
+ public static String spilled_minidump_storage_path = System.getenv("LOG_DIR") + File.separator + "minidump";
+
// The max number of profiles that can be stored to storage.
@ConfField
public static int max_spilled_profile_num = 500;
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index d1441dea2cf..47a45b67aa7 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -221,6 +221,7 @@ DOUBLE: 'DOUBLE';
DROP: 'DROP';
DROPP: 'DROPP';
DUAL: 'DUAL';
+DUMP: 'DUMP';
DUPLICATE: 'DUPLICATE';
DYNAMIC: 'DYNAMIC';
E:'E';
@@ -409,6 +410,7 @@ PHYSICAL: 'PHYSICAL';
PI: 'PI';
PLACEHOLDER: '?';
PLAN: 'PLAN';
+PLAY: 'PLAY';
PRIVILEGES: 'PRIVILEGES';
PROCESS: 'PROCESS';
PLUGIN: 'PLUGIN';
@@ -444,6 +446,7 @@ REPAIR: 'REPAIR';
REPEATABLE: 'REPEATABLE';
REPLACE: 'REPLACE';
REPLACE_IF_NOT_NULL: 'REPLACE_IF_NOT_NULL';
+REPLAYER: 'REPLAYER';
REPLICA: 'REPLICA';
REPOSITORIES: 'REPOSITORIES';
REPOSITORY: 'REPOSITORY';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 41c35cda988..0d39145c268 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -149,6 +149,7 @@ supportedDmlStatement
TO filePath=STRING_LITERAL
(propertyClause)?
(withRemoteStorageSystem)? #export
+ | replayCommand #replay
;
supportedCreateStatement
@@ -983,7 +984,7 @@ grantUserIdentify
explain
: explainCommand planType?
- level=(VERBOSE | TREE | GRAPH | PLAN)?
+ level=(VERBOSE | TREE | GRAPH | PLAN | DUMP)?
PROCESS?
;
@@ -1004,6 +1005,13 @@ planType
| ALL // default type
;
+replayCommand
+ : PLAN REPLAYER replayType;
+
+replayType
+ : DUMP query
+ | PLAY filePath=STRING_LITERAL;
+
mergeType
: APPEND
| DELETE
@@ -2014,6 +2022,7 @@ nonReserved
| REPEATABLE
| REPLACE
| REPLACE_IF_NOT_NULL
+ | REPLAYER
| REPOSITORIES
| REPOSITORY
| RESOURCE
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtType.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtType.java
index 426233ff7b4..a48dc45f6eb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtType.java
@@ -43,6 +43,7 @@ public enum StmtType {
PREPARE,
RECOVER,
REFRESH,
+ REPLAY,
RESTORE,
RESUME,
REVOKE,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java
index 72ed921633a..918fc5c7db8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java
@@ -25,6 +25,7 @@ import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Writable;
import com.google.common.collect.Lists;
+import com.google.gson.annotations.SerializedName;
import java.io.DataInput;
import java.io.DataOutput;
@@ -37,9 +38,13 @@ import java.util.stream.Collectors;
* This class saves the schema of a colocation group
*/
public class ColocateGroupSchema implements Writable {
+ @SerializedName(value = "groupId")
private GroupId groupId;
+ @SerializedName(value = "distributionColTypes")
private List<Type> distributionColTypes = Lists.newArrayList();
+ @SerializedName(value = "bucketsNum")
private int bucketsNum;
+ @SerializedName(value = "replicaAlloc")
private ReplicaAllocation replicaAlloc;
private ColocateGroupSchema() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java
index d253e4bae8c..dae95698807 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateTableIndex.java
@@ -169,18 +169,25 @@ public class ColocateTableIndex implements Writable {
}
// group_name -> group_id
+ @SerializedName(value = "groupName2Id")
private Map<String, GroupId> groupName2Id = Maps.newHashMap();
// group_id -> table_ids
+ @SerializedName(value = "group2Tables")
private Multimap<GroupId, Long> group2Tables = ArrayListMultimap.create();
// table_id -> group_id
+ @SerializedName(value = "table2Group")
private Map<Long, GroupId> table2Group = Maps.newHashMap();
// group id -> group schema
+ @SerializedName(value = "group2Schema")
private Map<GroupId, ColocateGroupSchema> group2Schema = Maps.newHashMap();
// group_id -> bucketSeq -> backend ids
+ @SerializedName(value = "group2BackendsPerBucketSeq")
private Table<GroupId, Tag, List<List<Long>>> group2BackendsPerBucketSeq = HashBasedTable.create();
// the colocate group is unstable
+ @SerializedName(value = "unstableGroups")
private Set<GroupId> unstableGroups = Sets.newHashSet();
// save some error msg of the group for show. no need to persist
+ @SerializedName(value = "group2ErrMsgs")
private Map<GroupId, String> group2ErrMsgs = Maps.newHashMap();
private transient MonitoredReentrantReadWriteLock lock = new MonitoredReentrantReadWriteLock();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
index e9392deae4a..234128582fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -78,6 +78,7 @@ public abstract class Table extends MetaObject implements Writable, TableIf, Gso
protected long id;
@SerializedName(value = "name")
protected volatile String name;
+ @SerializedName(value = "qualifiedDbName")
protected volatile String qualifiedDbName;
@SerializedName(value = "type")
protected TableType type;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java
index 236a1f0fc28..69f8b5ebeca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java
@@ -148,6 +148,10 @@ public class View extends Table implements GsonPostProcessable {
this.sqlMode = sqlMode;
}
+ public long getSqlMode() {
+ return sqlMode;
+ }
+
public String getInlineViewDef() {
return inlineViewDef;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java
index fcbeecb6ee8..eb75bc9312a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/FrontendsProcNode.java
@@ -171,6 +171,15 @@ public class FrontendsProcNode implements ProcNodeInterface {
}
}
+ public static Frontend getCurrentFrontendVersion(Env env) {
+ for (Frontend fe : env.getFrontends(null /* all */)) {
+ if (fe.getHost().equals(env.getSelfNode().getHost())) {
+ return fe;
+ }
+ }
+ return null;
+ }
+
public static void getFrontendsDiskInfo(Env env, List<List<String>> infos) {
for (Frontend fe : env.getFrontends(null /* all */)) {
if (fe.getDiskInfos() != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MinidumpAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MinidumpAction.java
new file mode 100644
index 00000000000..1cc5cf46644
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MinidumpAction.java
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.httpv2.rest;
+
+import org.apache.doris.httpv2.entity.ResponseEntityBuilder;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.minidump.MinidumpUtils;
+import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Strings;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+// This class is a RESTFUL interface to get query minidump.
+// It will be used in query monitor to collect minidumps.
+// Usage:
+// wget http://fe_host:fe_http_port/api/minidump?query_id=123456
+@RestController
+public class MinidumpAction extends RestBaseController {
+ private static final Logger LOG = LogManager.getLogger(MinidumpAction.class);
+
+ @RequestMapping(path = "/api/minidump", method = RequestMethod.GET)
+ protected Object minidump(HttpServletRequest request, HttpServletResponse response) {
+ executeCheckPassword(request, response);
+ checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN);
+
+ String queryId = request.getParameter("query_id");
+ if (Strings.isNullOrEmpty(queryId)) {
+ return ResponseEntityBuilder.badRequest("Missing query_id");
+ }
+
+ String queryMinidumpStr = MinidumpUtils.getMinidumpString(queryId);
+ if (queryMinidumpStr == null) {
+ return ResponseEntityBuilder.okWithCommonError("query id " + queryId + " not found.");
+ }
+
+ // Set the headers to indicate a file attachment and set the filename
+ HttpHeaders headers = new HttpHeaders();
+ headers.setContentType(MediaType.APPLICATION_JSON);
+ headers.setContentDispositionFormData("attachment", queryId + ".json");
+
+ // Return the file data as a byte array along with headers
+ return ResponseEntity.ok()
+ .headers(headers)
+ .body(queryMinidumpStr);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
index 66916744262..73f3ceb385f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
@@ -67,9 +67,7 @@ import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.statistics.StatisticsBuilder;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import org.apache.logging.log4j.LogManager;
@@ -114,7 +112,7 @@ public class CascadesContext implements ScheduleContext {
private final RuntimeFilterContext runtimeFilterContext;
private final TopnFilterContext topnFilterContext = new TopnFilterContext();
private Optional<Scope> outerScope = Optional.empty();
- private Map<Long, TableIf> tables = null;
+ private Map<List<String>, TableIf> tables = null;
private boolean isRewriteRoot;
private volatile boolean isTimeout = false;
@@ -266,9 +264,8 @@ public class CascadesContext implements ScheduleContext {
this.memo = null;
}
- public void setTables(List<TableIf> tables) {
- this.tables = tables.stream()
- .collect(Collectors.toMap(TableIf::getId, t -> t, (t1, t2) -> t1, () -> Maps.newTreeMap()));
+ public void setTables(Map<List<String>, TableIf> tables) {
+ this.tables = tables;
}
public final ConnectContext getConnectContext() {
@@ -406,11 +403,11 @@ public class CascadesContext implements ScheduleContext {
*/
public void extractTables(LogicalPlan logicalPlan) {
Set<List<String>> tableNames = getTables(logicalPlan);
- tables = Maps.newTreeMap();
+ tables = Maps.newHashMap();
for (List<String> tableName : tableNames) {
try {
TableIf table = getTable(tableName);
- tables.put(table.getId(), table);
+ tables.put(table.getFullQualifiers(), table);
} catch (Throwable e) {
// IGNORE
}
@@ -418,25 +415,11 @@ public class CascadesContext implements ScheduleContext {
}
- /** get table by table name, try to get from information from dumpfile first */
- public TableIf getTableInMinidumpCache(String tableName) {
- Preconditions.checkState(tables != null, "tables should not be null");
- for (TableIf table : tables.values()) {
- if (table.getName().equals(tableName)) {
- return table;
- }
- }
- if (getConnectContext().getSessionVariable().isPlayNereidsDump()) {
- throw new AnalysisException("Minidump cache can not find table:" + tableName);
- }
- return null;
- }
-
- public List<TableIf> getTables() {
+ public Map<List<String>, TableIf> getTables() {
if (tables == null) {
return null;
} else {
- return Lists.newArrayList(tables.values());
+ return tables;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 25d0dedd58d..6cbf9dda588 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -531,7 +531,7 @@ public class NereidsPlanner extends Planner {
String used = "";
String unUsed = "";
String syntaxError = "";
- int distributeHintIndex = 1;
+ int distributeHintIndex = 0;
for (Hint hint : hints) {
String distributeIndex = "";
if (hint instanceof DistributeHint) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
index 943a404783c..f6cf30d855f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
@@ -74,11 +74,7 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> {
public CostModelV1(ConnectContext connectContext) {
SessionVariable sessionVariable = connectContext.getSessionVariable();
- if (sessionVariable.isPlayNereidsDump()) {
- // TODO: @bingfeng refine minidump setting, and pass testMinidumpUt
- beNumber = 1;
- parallelInstance = Math.max(1, connectContext.getSessionVariable().getParallelExecInstanceNum());
- } else if (sessionVariable.getBeNumberForTest() != -1) {
+ if (sessionVariable.getBeNumberForTest() != -1) {
// shape test, fix the BE number and instance number
beNumber = sessionVariable.getBeNumberForTest();
parallelInstance = 8;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJob.java
index 8aa940589f8..dd1299eec7c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJob.java
@@ -26,6 +26,7 @@ import org.apache.doris.nereids.metrics.EventChannel;
import org.apache.doris.nereids.metrics.EventProducer;
import org.apache.doris.nereids.metrics.consumer.LogConsumer;
import org.apache.doris.nereids.metrics.event.StatsStateEvent;
+import org.apache.doris.nereids.minidump.MinidumpUtils;
import org.apache.doris.nereids.stats.StatsCalculator;
import org.apache.doris.nereids.trees.expressions.CTEId;
import org.apache.doris.nereids.trees.plans.algebra.Project;
@@ -113,7 +114,7 @@ public class DeriveStatsJob extends Job {
context.getCascadesContext());
STATS_STATE_TRACER.log(StatsStateEvent.of(groupExpression,
groupExpression.getOwnerGroup().getStatistics()));
- if (sessionVariable.isEnableMinidump() && !sessionVariable.isPlayNereidsDump()) {
+ if (MinidumpUtils.isDump() && !sessionVariable.isPlayNereidsDump()) {
connectContext.getTotalColumnStatisticMap().putAll(statsCalculator.getTotalColumnStatisticMap());
connectContext.getTotalHistogramMap().putAll(statsCalculator.getTotalHistogramMap());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java
index 2b6276b0193..5c324e1f364 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/Minidump.java
@@ -19,6 +19,9 @@ package org.apache.doris.nereids.minidump;
import org.apache.doris.catalog.ColocateTableIndex;
import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.OriginStatement;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.Histogram;
@@ -48,7 +51,7 @@ public class Minidump {
private String dbName;
// metadata objects
- private List<TableIf> tables;
+ private Map<List<String>, TableIf> tables;
private Map<String, ColumnStatistic> totalColumnStatisticMap = new HashMap<>();
@@ -58,7 +61,7 @@ public class Minidump {
/** Minidump class used to save environment messages */
public Minidump(String sql, SessionVariable sessionVariable,
- String parsedPlanJson, String resultPlanJson, List<TableIf> tables,
+ String parsedPlanJson, String resultPlanJson, Map<List<String>, TableIf> tables,
String dbName, Map<String, ColumnStatistic> totalColumnStatisticMap,
Map<String, Histogram> totalHistogramMap, ColocateTableIndex colocateTableIndex) {
this.sql = sql;
@@ -84,7 +87,7 @@ public class Minidump {
return resultPlanJson;
}
- public List<TableIf> getTables() {
+ public Map<List<String>, TableIf> getTables() {
return tables;
}
@@ -111,7 +114,16 @@ public class Minidump {
/** Nereids minidump entry, argument should be absolute address of minidump path */
public static void main(String[] args) {
assert (args.length == 1);
- Minidump minidump = MinidumpUtils.loadMinidumpInputs(args[0]);
+ Minidump minidump = null;
+ try {
+ minidump = MinidumpUtils.loadMinidumpInputs(args[0]);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ StatementContext statementContext = new StatementContext(ConnectContext.get(),
+ new OriginStatement(minidump.getSql(), 0));
+ ConnectContext.get().setStatementContext(statementContext);
JSONObject resultPlan = MinidumpUtils.executeSql(minidump.getSql());
JSONObject minidumpResult = new JSONObject(minidump.getResultPlanJson());
@@ -121,9 +133,10 @@ public class Minidump {
System.out.println(minidump.sql);
} else {
System.out.println(minidump.sql + "\n" + "Result plan are not expected. Differences:");
- for (String difference : differences) {
- System.out.println(difference + "\n");
- }
+ System.out.println("----------------------------Result plan--------------------------");
+ System.out.println(resultPlan + "\n");
+ System.out.println("----------------------------Minidump plan--------------------------");
+ System.out.println(minidumpResult + "\n");
}
System.exit(0);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java
index 8d6d9b0e794..fad7befc162 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java
@@ -23,6 +23,10 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.SchemaTable;
import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.View;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.proc.FrontendsProcNode;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.StatementContext;
@@ -41,6 +45,8 @@ import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ColumnStatisticBuilder;
import org.apache.doris.statistics.Histogram;
+import com.google.common.collect.Maps;
+import com.google.gson.reflect.TypeToken;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONArray;
@@ -51,6 +57,9 @@ import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Field;
+import java.lang.reflect.Type;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -65,15 +74,43 @@ public class MinidumpUtils {
private static final Logger LOG = LogManager.getLogger(MinidumpUtils.class);
- private static String DUMP_PATH = null;
+ private static String DUMP_PATH = Config.spilled_minidump_storage_path;
+
+ private static String DUMP_FILE_FULL_PATH = null;
+
+ private static String HTTP_GET_STRING = null;
+
+ private static boolean dump = false;
+
+ private static final int FE_VERSION_PREFIX_LENGTH = 13;
+
+ public static void openDump() {
+ dump = true;
+ }
+
+ public static boolean isDump() {
+ return dump;
+ }
+
+ public static String getDumpFileFullPath() {
+ return DUMP_FILE_FULL_PATH;
+ }
+
+ public static String getHttpGetString() {
+ return HTTP_GET_STRING;
+ }
/**
* Saving of minidump file to fe log path
*/
- public static void saveMinidumpString(JSONObject minidump, String dumpName) {
- String dumpPath = MinidumpUtils.DUMP_PATH + "/" + dumpName;
+ public static void saveMinidumpString(JSONObject minidump, String querId) {
+ String dumpPath = MinidumpUtils.DUMP_PATH + File.separator + "_" + querId;
+ String feAddress = FrontendsProcNode.getCurrentFrontendVersion(Env.getCurrentEnv()).getHost();
+ int feHttpPort = Config.http_port;
+ MinidumpUtils.DUMP_FILE_FULL_PATH = dumpPath + ".json";
+ MinidumpUtils.HTTP_GET_STRING = "http://" + feAddress + ":" + feHttpPort + "/api/minidump?query_id=" + querId;
String jsonMinidump = minidump.toString(4);
- try (FileWriter file = new FileWriter(dumpPath + ".json")) {
+ try (FileWriter file = new FileWriter(MinidumpUtils.DUMP_FILE_FULL_PATH)) {
file.write(jsonMinidump);
} catch (IOException e) {
LOG.info("failed to save minidump file", e);
@@ -83,8 +120,8 @@ public class MinidumpUtils {
/**
* deserialize of tables using gson, we need to get table type first
*/
- private static List<TableIf> dserializeTables(JSONArray tablesJson) {
- List<TableIf> tables = new ArrayList<>();
+ private static Map<List<String>, TableIf> dserializeTables(JSONArray tablesJson) {
+ Map<List<String>, TableIf> tables = Maps.newHashMap();
for (int i = 0; i < tablesJson.length(); i++) {
JSONObject tableJson = (JSONObject) tablesJson.get(i);
TableIf newTable;
@@ -93,11 +130,19 @@ public class MinidumpUtils {
String tableJsonValue = tableJson.get("TableValue").toString();
newTable = GsonUtils.GSON.fromJson(tableJsonValue, OlapTable.class);
break;
+ case "VIEW":
+ String viewJsonValue = tableJson.get("TableValue").toString();
+ newTable = GsonUtils.GSON.fromJson(viewJsonValue, View.class);
+ ((View) newTable).setInlineViewDefWithSqlMode(tableJson.get("InlineViewDef").toString(),
+ tableJson.getLong("SqlMode"));
+ break;
default:
newTable = null;
break;
}
- tables.add(newTable);
+ Type listType = new TypeToken<List<String>>() {}.getType();
+ List<String> key = GsonUtils.GSON.fromJson(tableJson.getString("TableName"), listType);
+ tables.put(key, newTable);
}
return tables;
}
@@ -105,15 +150,22 @@ public class MinidumpUtils {
/**
* Load minidump to memory using string
*/
- public static Minidump jsonMinidumpLoadFromString(String inputString) throws IOException {
+ public static Minidump jsonMinidumpLoadFromString(String inputString) throws Exception {
// Parse the JSON string back into a JSON object
JSONObject inputJSON = new JSONObject(inputString);
+ String dumpFeVersion = inputJSON.getString("FeVersion");
+ String currFeVersion = FrontendsProcNode.getCurrentFrontendVersion(Env.getCurrentEnv()).getVersion()
+ .substring(FE_VERSION_PREFIX_LENGTH);
+ if (!currFeVersion.equals(dumpFeVersion)) {
+ throw new AnalysisException("fe version:" + currFeVersion
+ + " does not match dump fe version: " + dumpFeVersion);
+ }
SessionVariable newSessionVariable = new SessionVariable();
newSessionVariable.readFromJson(inputJSON.getString("SessionVariable"));
String sql = inputJSON.getString("Sql");
JSONArray tablesJson = (JSONArray) inputJSON.get("Tables");
- List<TableIf> tables = dserializeTables(tablesJson);
+ Map<List<String>, TableIf> tables = dserializeTables(tablesJson);
String colocateTableIndexJson = inputJSON.get("ColocateTableIndex").toString();
ColocateTableIndex newColocateTableIndex
@@ -125,7 +177,7 @@ public class MinidumpUtils {
JSONObject oneColumnStat = (JSONObject) columnStats.get(i);
String colName = oneColumnStat.keys().next();
String colStat = oneColumnStat.getString(colName);
- ColumnStatistic columnStatistic = ColumnStatistic.fromJson(colStat);
+ ColumnStatistic columnStatistic = GsonUtils.GSON.fromJson(colStat, ColumnStatistic.class);
columnStatisticMap.put(colName, columnStatistic);
}
JSONArray histogramJsonArray = (JSONArray) inputJSON.get("Histogram");
@@ -148,7 +200,7 @@ public class MinidumpUtils {
/**
* Loading of minidump file
*/
- public static Minidump jsonMinidumpLoad(String dumpFilePath) throws IOException {
+ public static Minidump jsonMinidumpLoad(String dumpFilePath) throws Exception {
// open file, read file, put them into minidump object
try (FileInputStream inputStream = new FileInputStream(dumpFilePath)) {
StringBuilder sb = new StringBuilder();
@@ -177,7 +229,6 @@ public class MinidumpUtils {
connectContext.setSessionVariable(minidump.getSessionVariable());
connectContext.setTables(minidump.getTables());
connectContext.setDatabase(minidump.getDbName());
- connectContext.getSessionVariable().setEnableMinidump(false);
connectContext.getSessionVariable().setPlanNereidsDump(true);
connectContext.getSessionVariable().enableNereidsTimeout = false;
}
@@ -187,15 +238,19 @@ public class MinidumpUtils {
* @param minidumpPath path of minidump file
* @return minidump messages in memory
*/
- public static Minidump loadMinidumpInputs(String minidumpPath) {
+ public static Minidump loadMinidumpInputs(String minidumpPath) throws AnalysisException {
Minidump minidump = null;
try {
minidump = MinidumpUtils.jsonMinidumpLoad(minidumpPath);
- } catch (IOException e) {
+ } catch (AnalysisException e) {
+ throw e;
+ } catch (Exception e) {
throw new RuntimeException(e);
}
setConnectContext(minidump);
+ Env env = Env.getCurrentEnv();
+ ConnectContext.get().setEnv(env);
ConnectContext.get().getSessionVariable().setEnableNereidsTrace(false);
ConnectContext.get().getSessionVariable().setNereidsTraceEventMode("all");
return minidump;
@@ -221,13 +276,17 @@ public class MinidumpUtils {
/**
* serialize tables from Table in catalog to json format
*/
- public static JSONArray serializeTables(List<TableIf> tables) {
+ public static JSONArray serializeTables(Map<List<String>, TableIf> tables) {
JSONArray tablesJson = new JSONArray();
- for (TableIf table : tables) {
- String tableValues = GsonUtils.GSON.toJson(table);
+ for (Map.Entry<List<String>, TableIf> table : tables.entrySet()) {
+ String tableValues = GsonUtils.GSON.toJson(table.getValue());
JSONObject oneTableJson = new JSONObject();
- oneTableJson.put("TableType", table.getType());
- oneTableJson.put("TableName", table.getName());
+ oneTableJson.put("TableType", table.getValue().getType());
+ oneTableJson.put("TableName", GsonUtils.GSON.toJson(table.getKey()));
+ if (table.getValue() instanceof View) {
+ oneTableJson.put("InlineViewDef", ((View) table.getValue()).getInlineViewDef());
+ oneTableJson.put("SqlMode", Long.toString(((View) table.getValue()).getSqlMode()));
+ }
JSONObject jsonTableValues = new JSONObject(tableValues);
oneTableJson.put("TableValue", jsonTableValues);
tablesJson.put(oneTableJson);
@@ -254,10 +313,11 @@ public class MinidumpUtils {
/**
* serialize column statistic and histograms when loading to dumpfile and environment
*/
- private static void serializeStatsUsed(JSONObject jsonObj, List<TableIf> tables) {
+ private static void serializeStatsUsed(JSONObject jsonObj, Map<List<String>, TableIf> tables) {
JSONArray columnStatistics = new JSONArray();
JSONArray histograms = new JSONArray();
- for (TableIf table : tables) {
+ for (Map.Entry<List<String>, TableIf> tableEntry : tables.entrySet()) {
+ TableIf table = tableEntry.getValue();
if (table instanceof SchemaTable) {
continue;
}
@@ -276,11 +336,11 @@ public class MinidumpUtils {
Histogram histogram = getColumnHistogram(table, colName);
if (histogram != null) {
JSONObject oneHistogram = new JSONObject();
- oneHistogram.put(colName, Histogram.serializeToJson(histogram));
+ oneHistogram.put(table.getName() + colName, Histogram.serializeToJson(histogram));
histograms.put(oneHistogram);
}
JSONObject oneColumnStats = new JSONObject();
- oneColumnStats.put(colName, cache.toJson());
+ oneColumnStats.put(table.getName() + colName, GsonUtils.GSON.toJson(cache));
columnStatistics.put(oneColumnStats);
}
}
@@ -293,12 +353,11 @@ public class MinidumpUtils {
*/
public static void serializeOutputToDumpFile(Plan resultPlan) {
ConnectContext connectContext = ConnectContext.get();
- if (connectContext.getSessionVariable().isPlayNereidsDump()
- || !connectContext.getSessionVariable().isEnableMinidump()) {
+ if (!isDump()) {
return;
}
connectContext.getMinidump().put("ResultPlan", ((AbstractPlan) resultPlan).toJson());
- if (connectContext.getSessionVariable().isEnableMinidump()) {
+ if (isDump()) {
saveMinidumpString(connectContext.getMinidump(), DebugUtil.printId(connectContext.queryId()));
}
}
@@ -386,10 +445,39 @@ public class MinidumpUtils {
return differences;
}
+ private static String getValue(Object obj, Field field) {
+ try {
+ switch (field.getType().getSimpleName()) {
+ case "boolean":
+ return Boolean.toString(field.getBoolean(obj));
+ case "byte":
+ return Byte.toString(field.getByte(obj));
+ case "short":
+ return Short.toString(field.getShort(obj));
+ case "int":
+ return Integer.toString(field.getInt(obj));
+ case "long":
+ return Long.toString(field.getLong(obj));
+ case "float":
+ return Float.toString(field.getFloat(obj));
+ case "double":
+ return Double.toString(field.getDouble(obj));
+ case "String":
+ return (String) field.get(obj);
+ default:
+ return "";
+ }
+ } catch (IllegalAccessException e) {
+ LOG.warn("Access failed.", e);
+ }
+ return "";
+ }
+
/**
* serialize sessionVariables different than default
*/
- private static JSONObject serializeChangedSessionVariable(SessionVariable sessionVariable) throws IOException {
+ private static JSONObject serializeChangedSessionVariable(SessionVariable sessionVariable,
+ SessionVariable newVar) throws IOException {
JSONObject root = new JSONObject();
try {
for (Field field : SessionVariable.class.getDeclaredFields()) {
@@ -398,7 +486,7 @@ public class MinidumpUtils {
continue;
}
field.setAccessible(true);
- if (field.get(sessionVariable).equals(field.get(VariableMgr.getDefaultSessionVariable()))) {
+ if (getValue(sessionVariable, field).equals(getValue(newVar, field))) {
continue;
}
switch (field.getType().getSimpleName()) {
@@ -434,15 +522,22 @@ public class MinidumpUtils {
/**
* implementation of interface serializeInputsToDumpFile
*/
- private static JSONObject serializeInputs(Plan parsedPlan, List<TableIf> tables) throws IOException {
+ private static JSONObject serializeInputs(Plan parsedPlan, Map<List<String>, TableIf> tables) throws IOException {
ConnectContext connectContext = ConnectContext.get();
// Create a JSON object
JSONObject jsonObj = new JSONObject();
- jsonObj.put("Sql", connectContext.getStatementContext().getOriginStatement().originStmt);
+ String feVersion = FrontendsProcNode.getCurrentFrontendVersion(Env.getCurrentEnv()).getVersion()
+ .substring(FE_VERSION_PREFIX_LENGTH);
+ jsonObj.put("FeVersion", feVersion);
+ String sql = connectContext.getStatementContext().getOriginStatement().originStmt;
+ String sqlWithOutReplayCommand = sql.substring(14);
+ jsonObj.put("Sql", sqlWithOutReplayCommand);
// add session variable
int beNumber = connectContext.getEnv().getClusterInfo().getBackendsNumber(true);
connectContext.getSessionVariable().setBeNumberForTest(beNumber);
- jsonObj.put("SessionVariable", serializeChangedSessionVariable(connectContext.getSessionVariable()));
+ SessionVariable newVar = new SessionVariable();
+ jsonObj.put("SessionVariable", serializeChangedSessionVariable(connectContext.getSessionVariable(), newVar));
+ jsonObj.put("CatalogMgr", GsonUtils.GSON.toJson(Env.getCurrentEnv().getInternalCatalog()));
// add tables
jsonObj.put("DbName", connectContext.getDatabase());
JSONArray tablesJson = serializeTables(tables);
@@ -463,23 +558,48 @@ public class MinidumpUtils {
* @param tables all tables relative to this query
* @throws IOException this will write to disk, so io exception should be dealed with
*/
- public static void serializeInputsToDumpFile(Plan parsedPlan, List<TableIf> tables) throws IOException {
+ public static void serializeInputsToDumpFile(Plan parsedPlan, Map<List<String>, TableIf> tables)
+ throws IOException {
ConnectContext connectContext = ConnectContext.get();
// when playing minidump file, we do not save input again.
- if (connectContext.getSessionVariable().isPlayNereidsDump()
- || !connectContext.getSessionVariable().isEnableMinidump()) {
+ if (!isDump()) {
return;
}
- if (!connectContext.getSessionVariable().getMinidumpPath().equals("")) {
- MinidumpUtils.DUMP_PATH = connectContext.getSessionVariable().getMinidumpPath();
- } else {
- connectContext.getSessionVariable().setMinidumpPath("defaultMinidumpPath");
- }
MinidumpUtils.init();
connectContext.setMinidump(serializeInputs(parsedPlan, tables));
}
+ /**
+ * get minidump string by query id, would find file in DUMP_PATH
+ * @param queryId unique query id of a sql
+ * @return minidump file content
+ */
+ public static String getMinidumpString(String queryId) {
+ // Create a File object for the directory
+ File directory = new File(DUMP_PATH);
+
+ // Get all files in the directory
+ File[] files = directory.listFiles();
+
+ if (files != null) {
+ for (File file : files) {
+ // Check if the file name contains the search string
+ if (file.isFile() && file.getName().contains(queryId)) {
+ try {
+ // Read the content of the file
+ String content = new String(Files.readAllBytes(Paths.get(file.getPath())));
+ // Add the content to the list
+ return content;
+ } catch (IOException e) {
+ break;
+ }
+ }
+ }
+ }
+ return null;
+ }
+
/**
* init minidump utils before start to dump file, this will create a path
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index fbee3b858ba..82045b8dac2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -361,6 +361,7 @@ import org.apache.doris.nereids.trees.plans.DistributeType;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.LimitPhase;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
@@ -391,6 +392,7 @@ import org.apache.doris.nereids.trees.plans.commands.ExportCommand;
import org.apache.doris.nereids.trees.plans.commands.LoadCommand;
import org.apache.doris.nereids.trees.plans.commands.PauseMTMVCommand;
import org.apache.doris.nereids.trees.plans.commands.RefreshMTMVCommand;
+import org.apache.doris.nereids.trees.plans.commands.ReplayCommand;
import org.apache.doris.nereids.trees.plans.commands.ResumeMTMVCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowConfigCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowConstraintsCommand;
@@ -774,6 +776,19 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return new MTMVRefreshTriggerInfo(RefreshTrigger.MANUAL);
}
+ @Override
+ public ReplayCommand visitReplay(DorisParser.ReplayContext ctx) {
+ if (ctx.replayCommand().replayType().DUMP() != null) {
+ LogicalPlan plan = plan(ctx.replayCommand().replayType().query());
+ return new ReplayCommand(PlanType.REPLAY_COMMAND, null, plan, ReplayCommand.ReplayType.DUMP);
+ } else if (ctx.replayCommand().replayType().PLAY() != null) {
+ String tmpPath = ctx.replayCommand().replayType().filePath.getText();
+ String path = LogicalPlanBuilderAssistant.escapeBackSlash(tmpPath.substring(1, tmpPath.length() - 1));
+ return new ReplayCommand(PlanType.REPLAY_COMMAND, path, null, ReplayCommand.ReplayType.PLAY);
+ }
+ return null;
+ }
+
@Override
public MTMVRefreshSchedule visitRefreshSchedule(RefreshScheduleContext ctx) {
int interval = Integer.parseInt(ctx.INTEGER_VALUE().getText());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 2d592a02e97..217280d7d49 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -171,14 +171,18 @@ public class BindRelation extends OneAnalysisRuleFactory {
List<String> tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(),
unboundRelation.getNameParts());
TableIf table = null;
- if (customTableResolver.isPresent()) {
- table = customTableResolver.get().apply(tableQualifier);
+ table = ConnectContext.get().getTableInMinidumpCache(tableQualifier);
+ if (table == null) {
+ if (customTableResolver.isPresent()) {
+ table = customTableResolver.get().apply(tableQualifier);
+ }
}
// In some cases even if we have already called the "cascadesContext.getTableByName",
// it also gets the null. So, we just check it in the catalog again for safety.
if (table == null) {
table = RelationUtil.getTable(tableQualifier, cascadesContext.getConnectContext().getEnv());
}
+ ConnectContext.get().getTables().put(tableQualifier, table);
// TODO: should generate different Scan sub class according to table's type
LogicalPlan scan = getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
@@ -191,18 +195,20 @@ public class BindRelation extends OneAnalysisRuleFactory {
}
private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboundRelation) {
- List<String> qualifiedTablName = RelationUtil.getQualifierName(cascadesContext.getConnectContext(),
+ List<String> tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(),
unboundRelation.getNameParts());
TableIf table = null;
if (customTableResolver.isPresent()) {
- table = customTableResolver.get().apply(qualifiedTablName);
+ table = customTableResolver.get().apply(tableQualifier);
}
+ table = ConnectContext.get().getTableInMinidumpCache(tableQualifier);
// In some cases even if we have already called the "cascadesContext.getTableByName",
// it also gets the null. So, we just check it in the catalog again for safety.
if (table == null) {
- table = RelationUtil.getTable(qualifiedTablName, cascadesContext.getConnectContext().getEnv());
+ table = RelationUtil.getTable(tableQualifier, cascadesContext.getConnectContext().getEnv());
}
- return getLogicalPlan(table, unboundRelation, qualifiedTablName, cascadesContext);
+ ConnectContext.get().getTables().put(tableQualifier, table);
+ return getLogicalPlan(table, unboundRelation, tableQualifier, cascadesContext);
}
private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List<String> qualifier) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 245d9ea2d69..73ea61fcfac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -168,5 +168,6 @@ public enum PlanType {
PREPARED_COMMAND,
EXECUTE_COMMAND,
- SHOW_CONFIG_COMMAND
+ SHOW_CONFIG_COMMAND,
+ REPLAY_COMMAND
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java
new file mode 100644
index 00000000000..15cc2b3696a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ReplayCommand.java
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.minidump.Minidump;
+import org.apache.doris.nereids.minidump.MinidumpUtils;
+import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.OriginStatement;
+import org.apache.doris.qe.StmtExecutor;
+
+import org.json.JSONObject;
+
+import java.util.List;
+
+/**
+ * replay command.
+ */
+public class ReplayCommand extends Command implements NoForward {
+
+ private final String dumpFileFullPath;
+
+ private final LogicalPlan plan;
+
+ private final ReplayType replayType;
+
+ public ReplayCommand(PlanType type, String dumpFileFullPath, LogicalPlan plan, ReplayType replayType) {
+ super(type);
+ this.dumpFileFullPath = dumpFileFullPath;
+ this.plan = plan;
+ this.replayType = replayType;
+ }
+
+ public String getDumpFileFullPath() {
+ return dumpFileFullPath;
+ }
+
+ public ReplayType getReplayType() {
+ return replayType;
+ }
+
+ /**
+ * explain level.
+ */
+ public enum ReplayType {
+ DUMP,
+ PLAY
+ }
+
+ @Override
+ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+ if (this.replayType == ReplayType.DUMP) {
+ handleDump(ctx, executor);
+ } else if (this.replayType == ReplayType.PLAY) {
+ handleLoad();
+ }
+ }
+
+ private void handleDump(ConnectContext ctx, StmtExecutor executor) throws Exception {
+ LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(plan, ctx.getStatementContext());
+ MinidumpUtils.openDump();
+ executor.setParsedStmt(logicalPlanAdapter);
+ NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
+ if (ctx.getSessionVariable().isEnableMaterializedViewRewrite()) {
+ ctx.getStatementContext().addPlannerHook(InitMaterializationContextHook.INSTANCE);
+ }
+ planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift());
+ executor.setPlanner(planner);
+ executor.checkBlockRules();
+ executor.handleReplayStmt(MinidumpUtils.getHttpGetString());
+ }
+
+ private void handleLoad() throws Exception {
+ // 1. check fe version, if not matched throw exception
+ // 2. load every thing from minidump file and replace original ones
+ Minidump minidump = MinidumpUtils.loadMinidumpInputs(dumpFileFullPath);
+ // 3. run nereids planner with sql in minidump file
+ StatementContext statementContext = new StatementContext(ConnectContext.get(),
+ new OriginStatement(minidump.getSql(), 0));
+ ConnectContext.get().setStatementContext(statementContext);
+ JSONObject resultPlan = MinidumpUtils.executeSql(minidump.getSql());
+ JSONObject minidumpResult = new JSONObject(minidump.getResultPlanJson());
+
+ List<String> differences = MinidumpUtils.compareJsonObjects(minidumpResult, resultPlan, "");
+ assert (differences.isEmpty());
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitReplayCommand(this, context);
+ }
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.REPLAY;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
index 15e06816c1a..ac986cbe77b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -65,6 +65,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
// Members for materialized index.
///////////////////////////////////////////////////////////////////////////
+ /**
/**
* The select materialized index id to read data from.
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java
index 56c18908ad6..5f4bb5aa5f4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java
@@ -233,7 +233,7 @@ public abstract class AbstractPhysicalJoin<
properties.put("HashJoinConjuncts", hashJoinConjuncts.toString());
properties.put("OtherJoinConjuncts", otherJoinConjuncts.toString());
properties.put("MarkJoinConjuncts", markJoinConjuncts.toString());
- properties.put("JoinHint", hint.toString());
+ properties.put("JoinHint", hint.getExplainString());
properties.put("MarkJoinSlotReference", markJoinSlotReference.toString());
physicalJoin.put("Properties", properties);
return physicalJoin;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 92b5ecbd9e1..f35e6f8a640 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -41,6 +41,7 @@ import org.apache.doris.nereids.trees.plans.commands.ExportCommand;
import org.apache.doris.nereids.trees.plans.commands.LoadCommand;
import org.apache.doris.nereids.trees.plans.commands.PauseMTMVCommand;
import org.apache.doris.nereids.trees.plans.commands.RefreshMTMVCommand;
+import org.apache.doris.nereids.trees.plans.commands.ReplayCommand;
import org.apache.doris.nereids.trees.plans.commands.ResumeMTMVCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowConfigCommand;
import org.apache.doris.nereids.trees.plans.commands.ShowConstraintsCommand;
@@ -62,6 +63,10 @@ public interface CommandVisitor<R, C> {
return visitCommand(explain, context);
}
+ default R visitReplayCommand(ReplayCommand replay, C context) {
+ return visitCommand(replay, context);
+ }
+
default R visitCreatePolicyCommand(CreatePolicyCommand createPolicy, C context) {
return visitCommand(createPolicy, context);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
index dd8ad13ae6f..ae2fff46835 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java
@@ -47,6 +47,7 @@ import org.apache.doris.analysis.LambdaFunctionCallExpr;
import org.apache.doris.analysis.LambdaFunctionExpr;
import org.apache.doris.analysis.LargeIntLiteral;
import org.apache.doris.analysis.LikePredicate;
+import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.MapLiteral;
import org.apache.doris.analysis.MatchPredicate;
import org.apache.doris.analysis.MaxLiteral;
@@ -310,6 +311,7 @@ public class GsonUtils {
.registerSubtype(StringLiteral.class, StringLiteral.class.getSimpleName())
.registerSubtype(IntLiteral.class, IntLiteral.class.getSimpleName())
.registerSubtype(LargeIntLiteral.class, LargeIntLiteral.class.getSimpleName())
+ .registerSubtype(LiteralExpr.class, LiteralExpr.class.getSimpleName())
.registerSubtype(DecimalLiteral.class, DecimalLiteral.class.getSimpleName())
.registerSubtype(FloatLiteral.class, FloatLiteral.class.getSimpleName())
.registerSubtype(NullLiteral.class, NullLiteral.class.getSimpleName())
@@ -589,7 +591,8 @@ public class GsonUtils {
private static final GsonBuilder GSON_BUILDER = new GsonBuilder()
.setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE)
.addSerializationExclusionStrategy(
- new HiddenAnnotationExclusionStrategy()).enableComplexMapKeySerialization()
+ new HiddenAnnotationExclusionStrategy()).serializeSpecialFloatingPointValues()
+ .enableComplexMapKeySerialization()
.addReflectionAccessFilter(ReflectionAccessFilter.BLOCK_INACCESSIBLE_JAVA)
.registerTypeHierarchyAdapter(Table.class, new GuavaTableAdapter())
// .registerTypeHierarchyAdapter(Expr.class, new ExprAdapter())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
index 33e86856eb5..579fdeff8e7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
@@ -55,6 +55,7 @@ import org.apache.doris.mysql.MysqlSslContext;
import org.apache.doris.mysql.ProxyMysqlChannel;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.stats.StatsErrorEstimator;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.plsql.Exec;
@@ -73,6 +74,7 @@ import org.apache.doris.thrift.TUniqueId;
import org.apache.doris.transaction.TransactionEntry;
import org.apache.doris.transaction.TransactionStatus;
+import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -270,7 +272,7 @@ public class ConnectContext {
// new planner
private Map<String, PreparedStatementContext> preparedStatementContextMap = Maps.newHashMap();
- private List<TableIf> tables = null;
+ private Map<List<String>, TableIf> tables = null;
private Map<String, ColumnStatistic> totalColumnStatisticMap = new HashMap<>();
@@ -431,14 +433,30 @@ public class ConnectContext {
return this.preparedStatementContextMap.get(stmtName);
}
- public List<TableIf> getTables() {
+ public Map<List<String>, TableIf> getTables() {
+ if (tables == null) {
+ tables = Maps.newHashMap();
+ }
return tables;
}
- public void setTables(List<TableIf> tables) {
+ public void setTables(Map<List<String>, TableIf> tables) {
this.tables = tables;
}
+ /** get table by table name, try to get from information from dumpfile first */
+ public TableIf getTableInMinidumpCache(List<String> tableQualifier) {
+ if (!getSessionVariable().isPlayNereidsDump()) {
+ return null;
+ }
+ Preconditions.checkState(tables != null, "tables should not be null");
+ TableIf table = tables.getOrDefault(tableQualifier, null);
+ if (getSessionVariable().isPlayNereidsDump() && table == null) {
+ throw new AnalysisException("Minidump cache can not find table:" + tableQualifier);
+ }
+ return table;
+ }
+
public void closeTxn() {
if (isTxnModel()) {
try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 4964be5e58e..530b78e273a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -435,8 +435,6 @@ public class SessionVariable implements Serializable, Writable {
public static final String JOIN_REORDER_TIME_LIMIT = "join_order_time_limit";
public static final String SHOW_USER_DEFAULT_ROLE = "show_user_default_role";
- public static final String ENABLE_MINIDUMP = "enable_minidump";
-
public static final String ENABLE_PAGE_CACHE = "enable_page_cache";
public static final String MINIDUMP_PATH = "minidump_path";
@@ -1621,9 +1619,6 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = DPHYPER_LIMIT)
public int dphyperLimit = 1000;
- @VariableMgr.VarAttr(name = ENABLE_MINIDUMP)
- public boolean enableMinidump = false;
-
@VariableMgr.VarAttr(
name = ENABLE_PAGE_CACHE,
@@ -1636,9 +1631,6 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = ENABLE_FOLD_NONDETERMINISTIC_FN)
public boolean enableFoldNondeterministicFn = false;
- @VariableMgr.VarAttr(name = MINIDUMP_PATH)
- public String minidumpPath = "";
-
@VariableMgr.VarAttr(name = TRACE_NEREIDS)
public boolean traceNereids = false;
@@ -4004,22 +3996,6 @@ public class SessionVariable implements Serializable, Writable {
return "";
}
- public boolean isEnableMinidump() {
- return enableMinidump;
- }
-
- public void setEnableMinidump(boolean enableMinidump) {
- this.enableMinidump = enableMinidump;
- }
-
- public String getMinidumpPath() {
- return minidumpPath;
- }
-
- public void setMinidumpPath(String minidumpPath) {
- this.minidumpPath = minidumpPath;
- }
-
public boolean isTraceNereids() {
return traceNereids;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 5a36875140f..8d06d8d8dd4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -2940,6 +2940,28 @@ public class StmtExecutor {
context.getState().setEof();
}
+ public void handleReplayStmt(String result) throws IOException {
+ ShowResultSetMetaData metaData = ShowResultSetMetaData.builder()
+ .addColumn(new Column("Plan Replayer dump url",
+ ScalarType.createVarchar(20)))
+ .build();
+ if (context.getConnectType() == ConnectType.MYSQL) {
+ sendMetaData(metaData);
+
+ // Send result set.
+ for (String item : result.split("\n")) {
+ serializer.reset();
+ serializer.writeLenEncodedString(item);
+ context.getMysqlChannel().sendOnePacket(serializer.toByteBuffer());
+ }
+ } else if (context.getConnectType() == ConnectType.ARROW_FLIGHT_SQL) {
+ context.getFlightSqlChannel()
+ .addResult(DebugUtil.printId(context.queryId()), context.getRunningQuery(), metaData, result);
+ context.setReturnResultFromLocal(true);
+ }
+ context.getState().setEof();
+ }
+
private void handleLoadStmt() {
try {
LoadStmt loadStmt = (LoadStmt) parsedStmt;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
index 9713d2d30e1..919d2018ba8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
@@ -22,6 +22,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.collect.Sets;
@@ -74,6 +75,7 @@ public class ColumnStatistic {
public final double minValue;
@SerializedName("maxValue")
public final double maxValue;
+ @SerializedName("isUnKnown")
public final boolean isUnKnown;
/*
@@ -81,9 +83,12 @@ public class ColumnStatistic {
but originalNdv is not. It is used to trace the change of a column's ndv through serials
of sql operators.
*/
+ @SerializedName("original")
public final ColumnStatistic original;
+ @SerializedName("minExpr")
public final LiteralExpr minExpr;
+ @SerializedName("maxExpr")
public final LiteralExpr maxExpr;
@SerializedName("updatedTime")
@@ -300,8 +305,10 @@ public class ColumnStatistic {
statistic.put("AvgSizeByte", avgSizeByte);
statistic.put("NumNulls", numNulls);
statistic.put("DataSize", dataSize);
- statistic.put("MinExpr", minExpr);
- statistic.put("MaxExpr", maxExpr);
+ statistic.put("MinExprValue", minExpr.getStringValue());
+ statistic.put("MinExprType", minExpr.getType());
+ statistic.put("MaxExprValue", maxExpr.getStringValue());
+ statistic.put("MaxExprType", maxExpr.getType());
statistic.put("IsUnKnown", isUnKnown);
statistic.put("Original", original);
statistic.put("LastUpdatedTime", updatedTime);
@@ -310,7 +317,7 @@ public class ColumnStatistic {
// MinExpr and MaxExpr serialize and deserialize is not complete
// Histogram is got by other place
- public static ColumnStatistic fromJson(String statJson) {
+ public static ColumnStatistic fromJson(String statJson) throws AnalysisException {
JSONObject stat = new JSONObject(statJson);
Double minValue;
switch (stat.getString("MinValueType")) {
@@ -340,6 +347,15 @@ public class ColumnStatistic {
default:
throw new RuntimeException(String.format("Min value does not get anytype"));
}
+ String lastUpdatedTime = "";
+ try {
+ lastUpdatedTime = stat.getString("LastUpdatedTime");
+ } catch (Exception e) {
+ LOG.warn("lastUpdateTimeIsEmpty", e.getMessage());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(e);
+ }
+ }
return new ColumnStatistic(
stat.getDouble("Count"),
stat.getDouble("Ndv"),
@@ -349,10 +365,12 @@ public class ColumnStatistic {
stat.getDouble("DataSize"),
minValue,
maxValue,
- null,
- null,
+ LiteralExpr.create(stat.getString("MinExprValue"),
+ GsonUtils.GSON.fromJson(stat.getString("MinExprType"), Type.class)),
+ LiteralExpr.create(stat.getString("MaxExprValue"),
+ GsonUtils.GSON.fromJson(stat.getString("MaxExprType"), Type.class)),
stat.getBoolean("IsUnKnown"),
- stat.getString("LastUpdatedTime")
+ lastUpdatedTime
);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/minidump/MinidumpUtTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/minidump/MinidumpUtTest.java
index 74c2a4c7488..60ff399fa0b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/minidump/MinidumpUtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/minidump/MinidumpUtTest.java
@@ -21,8 +21,6 @@ import org.json.JSONObject;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-import java.io.IOException;
-
/**
* Used for add unit test of minidump
* Input: minidump filter which describe table information and sessionVariables which different with default
@@ -40,7 +38,7 @@ class MinidumpUtTest {
String currentMinidumpPath = "/src/test/java/org/apache/doris/nereids/minidump/MinidumpUtTestData.json";
try {
minidump = MinidumpUtils.jsonMinidumpLoad(directory + currentMinidumpPath);
- } catch (IOException e) {
+ } catch (Exception e) {
throw new RuntimeException(e);
}
MinidumpUtils.setConnectContext(minidump);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
index b7f9dcae9db..c8a5364b710 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
@@ -33,6 +33,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
+import org.apache.doris.nereids.trees.plans.commands.ReplayCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
@@ -181,6 +182,20 @@ public class NereidsParserTest extends ParserTestBase {
Assertions.assertEquals(ExplainLevel.GRAPH, explainLevel);
}
+ @Test
+ public void testPlanReplayer() {
+ String sql = "plan replayer dump select `AD``D` from t1 where a = 1";
+ NereidsParser nereidsParser = new NereidsParser();
+ LogicalPlan logicalPlan = nereidsParser.parseSingle(sql);
+ ReplayCommand replayCommand = (ReplayCommand) logicalPlan;
+ Assertions.assertEquals(ReplayCommand.ReplayType.DUMP, replayCommand.getReplayType());
+ sql = "plan replayer play 'path'";
+ logicalPlan = nereidsParser.parseSingle(sql);
+ replayCommand = (ReplayCommand) logicalPlan;
+ Assertions.assertEquals(ReplayCommand.ReplayType.PLAY, replayCommand.getReplayType());
+ Assertions.assertEquals("path", replayCommand.getDumpFileFullPath());
+ }
+
@Test
public void testParseSQL() {
String sql = "select `AD``D` from t1 where a = 1;explain graph select `AD``D` from t1 where a = 1;";
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ReadLockTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ReadLockTest.java
index e31fa98af11..3e1752e41bc 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ReadLockTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ReadLockTest.java
@@ -30,9 +30,10 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
-import java.util.stream.Collectors;
public class ReadLockTest extends SSBTestBase {
@@ -49,9 +50,14 @@ public class ReadLockTest extends SSBTestBase {
);
CascadesContext cascadesContext = planner.getCascadesContext();
- List<TableIf> f = cascadesContext.getTables();
- Assertions.assertEquals(1, f.size());
- Assertions.assertEquals("supplier", f.stream().map(TableIf::getName).findFirst().get());
+ Map<List<String>, TableIf> f = cascadesContext.getTables();
+ Assertions.assertEquals(2, f.size());
+ Set<String> tableNames = new HashSet<>();
+ for (Map.Entry<List<String>, TableIf> entry : f.entrySet()) {
+ TableIf table = entry.getValue();
+ tableNames.add(table.getName());
+ }
+ Assertions.assertTrue(tableNames.contains("supplier"));
}
@Test
@@ -70,9 +76,12 @@ public class ReadLockTest extends SSBTestBase {
PhysicalProperties.ANY
);
CascadesContext cascadesContext = planner.getCascadesContext();
- List<TableIf> f = cascadesContext.getTables();
+ Map<List<String>, TableIf> f = cascadesContext.getTables();
Assertions.assertEquals(1, f.size());
- Assertions.assertEquals("supplier", f.stream().map(TableIf::getName).findFirst().get());
+ for (Map.Entry<List<String>, TableIf> entry : f.entrySet()) {
+ TableIf table = entry.getValue();
+ Assertions.assertEquals("supplier", table.getName());
+ }
}
@Test
@@ -85,9 +94,12 @@ public class ReadLockTest extends SSBTestBase {
PhysicalProperties.ANY
);
CascadesContext cascadesContext = planner.getCascadesContext();
- List<TableIf> f = cascadesContext.getTables();
+ Map<List<String>, TableIf> f = cascadesContext.getTables();
Assertions.assertEquals(1, f.size());
- Assertions.assertEquals("supplier", f.stream().map(TableIf::getName).findFirst().get());
+ for (Map.Entry<List<String>, TableIf> entry : f.entrySet()) {
+ TableIf table = entry.getValue();
+ Assertions.assertEquals("supplier", table.getName());
+ }
}
@Test
@@ -100,9 +112,13 @@ public class ReadLockTest extends SSBTestBase {
PhysicalProperties.ANY
);
CascadesContext cascadesContext = planner.getCascadesContext();
- List<TableIf> f = cascadesContext.getTables();
+ Map<List<String>, TableIf> f = cascadesContext.getTables();
Assertions.assertEquals(2, f.size());
- Set<String> tableNames = f.stream().map(TableIf::getName).collect(Collectors.toSet());
+ Set<String> tableNames = new HashSet<>();
+ for (Map.Entry<List<String>, TableIf> entry : f.entrySet()) {
+ TableIf table = entry.getValue();
+ tableNames.add(table.getName());
+ }
Assertions.assertTrue(tableNames.contains("supplier"));
Assertions.assertTrue(tableNames.contains("lineorder"));
}
@@ -119,9 +135,13 @@ public class ReadLockTest extends SSBTestBase {
PhysicalProperties.ANY
);
CascadesContext cascadesContext = planner.getCascadesContext();
- List<TableIf> f = cascadesContext.getTables();
+ Map<List<String>, TableIf> f = cascadesContext.getTables();
Assertions.assertEquals(2, f.size());
- Set<String> tableNames = f.stream().map(TableIf::getName).collect(Collectors.toSet());
+ Set<String> tableNames = new HashSet<>();
+ for (Map.Entry<List<String>, TableIf> entry : f.entrySet()) {
+ TableIf table = entry.getValue();
+ tableNames.add(table.getName());
+ }
Assertions.assertTrue(tableNames.contains("supplier"));
Assertions.assertTrue(tableNames.contains("lineorder"));
}
diff --git a/regression-test/data/nereids_p0/eliminate_outer_join/eliminate_outer_join.out b/regression-test/data/nereids_p0/eliminate_outer_join/eliminate_outer_join.out
index 63c635b85e9..235247efe84 100644
--- a/regression-test/data/nereids_p0/eliminate_outer_join/eliminate_outer_join.out
+++ b/regression-test/data/nereids_p0/eliminate_outer_join/eliminate_outer_join.out
@@ -13,7 +13,7 @@ PhysicalResultSink
------PhysicalOlapScan[partsupp]
Hint log:
-Used: [broadcast]_2
+Used: [broadcast]_1
UnUsed:
SyntaxError:
@@ -31,7 +31,7 @@ PhysicalResultSink
Hint log:
Used:
-UnUsed: [broadcast]_2
+UnUsed: [broadcast]_1
SyntaxError:
-- !3 --
@@ -48,7 +48,7 @@ PhysicalResultSink
Hint log:
Used:
-UnUsed: [broadcast]_2
+UnUsed: [broadcast]_1
SyntaxError:
-- !4 --
@@ -59,7 +59,7 @@ PhysicalResultSink
----PhysicalOlapScan[nation]
Hint log:
-Used: [broadcast]_2
+Used: [broadcast]_1
UnUsed:
SyntaxError:
@@ -73,7 +73,7 @@ PhysicalResultSink
----PhysicalOlapScan[supplier]
Hint log:
-Used: [broadcast]_2
+Used: [broadcast]_1
UnUsed:
SyntaxError:
@@ -89,7 +89,7 @@ PhysicalResultSink
----PhysicalOlapScan[partsupp]
Hint log:
-Used: [broadcast]_2
+Used: [broadcast]_1
UnUsed:
SyntaxError:
@@ -106,7 +106,7 @@ PhysicalResultSink
----PhysicalOlapScan[partsupp]
Hint log:
-Used: [broadcast]_2
+Used: [broadcast]_1
UnUsed:
SyntaxError:
@@ -123,7 +123,7 @@ PhysicalResultSink
----PhysicalOlapScan[partsupp]
Hint log:
-Used: [broadcast]_2
+Used: [broadcast]_1
UnUsed:
SyntaxError:
@@ -140,7 +140,7 @@ PhysicalResultSink
----PhysicalOlapScan[partsupp]
Hint log:
-Used: [broadcast]_2
+Used: [broadcast]_1
UnUsed:
SyntaxError:
diff --git a/regression-test/suites/nereids_p0/hint/test_leading.groovy b/regression-test/suites/nereids_p0/hint/test_leading.groovy
index 5c46e827d25..48d3d38fc81 100644
--- a/regression-test/suites/nereids_p0/hint/test_leading.groovy
+++ b/regression-test/suites/nereids_p0/hint/test_leading.groovy
@@ -939,42 +939,38 @@ suite("test_leading") {
// distribute hint + leading hint
// only distribute hint + single hint
- // used
- qt_select90_1 """explain shape plan select count(*) from t1 join [broadcast] t2 on c1 = c2;"""
// unused
explain {
sql """shape plan select count(*) from t1 right outer join [broadcast] t2 on c1 = c2;"""
- contains("UnUsed: [broadcast]_2")
+ contains("UnUsed: [broadcast]_1")
}
// only distribute hint + multi hints
- qt_select90_3 """explain shape plan select count(*) from t1 join [broadcast] t2 on c1 = c2 join[shuffle] t3 on c2 = c3;"""
explain {
sql """shape plan select count(*) from t1 right outer join [broadcast] t2 on c1 = c2 join[shuffle] t3 on c2 = c3;"""
- contains("UnUsed: [broadcast]_2")
+ contains("UnUsed: [broadcast]_1")
}
- qt_select90_5 """explain shape plan select count(*) from t1 join [broadcast] t2 on c1 = c2 right outer join[shuffle] t3 on c2 = c3;"""
explain {
sql """shape plan select count(*) from t1 join [shuffle] t2 on c1 = c2 right outer join[broadcast] t3 on c2 = c3;"""
- contains("UnUsed: [broadcast]_3")
+ contains("UnUsed: [broadcast]_2")
}
// leading + distribute hint outside leading + single hint
explain {
sql """shape plan select /*+ leading(t1 t2 t3) */ count(*) from t1 join [broadcast] t2 on c1 = c2 join[shuffle] t3 on c2 = c3;"""
- contains("UnUsed: [broadcast]_2 [shuffle]_3")
+ contains("UnUsed: [broadcast]_1 [shuffle]_2")
}
explain {
sql """shape plan select /*+ leading(t1 t2 t3) */ count(*) from t1 right outer join [broadcast] t2 on c1 = c2 join[shuffle] t3 on c2 = c3;"""
- contains("UnUsed: [broadcast]_2 [shuffle]_3")
+ contains("UnUsed: [broadcast]_1 [shuffle]_2")
}
explain {
sql """shape plan select /*+ leading(t1 t2 t3) */ count(*) from t1 join [broadcast] t2 on c1 = c2 right outer join[shuffle] t3 on c2 = c3;"""
- contains("UnUsed: [broadcast]_2 [shuffle]_3")
+ contains("UnUsed: [broadcast]_1 [shuffle]_2")
}
explain {
sql """shape plan select /*+ leading(t1 t2 t3) */ count(*) from t1 join [shuffle] t2 on c1 = c2 right outer join[broadcast] t3 on c2 = c3;"""
- contains("UnUsed: [shuffle]_2 [broadcast]_3")
+ contains("UnUsed: [shuffle]_1 [broadcast]_2")
}
// leading + distribute hint inside leading + single hint
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org