You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/04/11 07:20:07 UTC
[doris] branch master updated: [ehancement](stats) Stats preheating as FE booted (#18502)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1b70db54df [ehancement](stats) Stats preheating as FE booted (#18502)
1b70db54df is described below
commit 1b70db54dfd4b372392cf294894c2f830f6f96dd
Author: AKIRA <33...@users.noreply.github.com>
AuthorDate: Tue Apr 11 16:19:59 2023 +0900
[ehancement](stats) Stats preheating as FE booted (#18502)
1. Support prefetch some column stats when FE booted, it would load column stats that was got updated recently according to the comment of PR #18460 from @morrySnow
2. Refactor stats cache, split histogram cache from column stats, so that we could avoid some redundant query for column statistics table,for example, update the histogram or column stats only, in the previous implementation a united cache loader would send query request to both column stats table and histogram table,
3. Extract some common logic to StatsUtil
4. Remove some useless codes in unit tests, those codes is hard to maintaince and it's not a good idea for testing the accurracy of stats estimation according to the advise from @englefly
5. Add field type restriction when create analysis tasks to avoid unnecessary failure
---
.../main/java/org/apache/doris/catalog/Env.java | 3 +
.../doris/nereids/stats/StatsCalculator.java | 24 +-
.../apache/doris/statistics/AnalysisManager.java | 4 +-
.../doris/statistics/AnalysisTaskScheduler.java | 58 +-
.../apache/doris/statistics/BaseAnalysisTask.java | 33 +-
.../statistics/ColumnLevelStatisticCache.java | 52 -
.../apache/doris/statistics/ColumnStatistic.java | 4 +-
.../statistics/ColumnStatisticsCacheLoader.java | 66 +
.../doris/statistics/HistogramCacheLoader.java | 65 +
.../org/apache/doris/statistics/HistogramTask.java | 2 +-
.../apache/doris/statistics/HiveAnalysisTask.java | 2 +-
.../apache/doris/statistics/MVAnalysisTask.java | 2 +-
.../apache/doris/statistics/OlapAnalysisTask.java | 2 +-
.../doris/statistics/StatisticConstants.java | 5 +-
.../apache/doris/statistics/StatisticsCache.java | 141 +-
.../doris/statistics/StatisticsCacheKey.java | 12 +
.../doris/statistics/StatisticsCacheLoader.java | 112 +-
.../doris/statistics/StatisticsRepository.java | 25 +-
.../doris/statistics/util/StatisticsUtil.java | 17 +
.../nereids/jobs/cascades/DeriveStatsJobTest.java | 3 -
.../org/apache/doris/nereids/stats/TPCHStats.java | 2094 --------------------
.../org/apache/doris/nereids/stats/TestStats.java | 151 --
.../org/apache/doris/statistics/CacheTest.java | 52 +-
23 files changed, 451 insertions(+), 2478 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 9ac1027efc..5bdf74f1fe 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -1345,6 +1345,9 @@ public class Env {
LOG.info(msg);
// for master, there are some new thread pools need to register metric
ThreadPoolManager.registerAllThreadPoolMetric();
+ if (analysisManager != null) {
+ analysisManager.getStatisticsCache().preHeat();
+ }
}
/*
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 942c19a6ac..75b637890d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -90,9 +90,9 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
import org.apache.doris.nereids.trees.plans.physical.PhysicalWindow;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import org.apache.doris.nereids.types.DataType;
-import org.apache.doris.statistics.ColumnLevelStatisticCache;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ColumnStatisticBuilder;
+import org.apache.doris.statistics.Histogram;
import org.apache.doris.statistics.StatisticRange;
import org.apache.doris.statistics.Statistics;
import org.apache.doris.statistics.StatisticsBuilder;
@@ -177,7 +177,6 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
@Override
public Statistics visitLogicalOlapScan(LogicalOlapScan olapScan, Void context) {
- olapScan.getExpressions();
return computeScan(olapScan);
}
@@ -431,17 +430,22 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
for (SlotReference slotReference : slotSet) {
String colName = slotReference.getName();
if (colName == null) {
- throw new RuntimeException(String.format("Column %s not found", colName));
+ throw new RuntimeException(String.format("Invalid slot: %s", slotReference.getExprId()));
}
- ColumnLevelStatisticCache cache =
- Env.getCurrentEnv().getStatisticsCache().getColumnStatistics(table.getId(), -1, colName);
- if (cache == null || cache.columnStatistic == null) {
- columnStatisticMap.put(slotReference, ColumnStatistic.UNKNOWN);
+ ColumnStatistic cache =
+ Env.getCurrentEnv().getStatisticsCache().getColumnStatistics(table.getId(), colName);
+ if (cache == ColumnStatistic.UNKNOWN) {
+ columnStatisticMap.put(slotReference, cache);
continue;
}
- ColumnStatisticBuilder columnStatisticBuilder =
- new ColumnStatisticBuilder(cache.columnStatistic).setHistogram(cache.getHistogram());
- columnStatisticMap.put(slotReference, columnStatisticBuilder.build());
+ Histogram histogram = Env.getCurrentEnv().getStatisticsCache().getHistogram(table.getId(), colName);
+ if (histogram != null) {
+ ColumnStatisticBuilder columnStatisticBuilder =
+ new ColumnStatisticBuilder(cache).setHistogram(histogram);
+ columnStatisticMap.put(slotReference, columnStatisticBuilder.build());
+ cache = columnStatisticBuilder.build();
+ }
+ columnStatisticMap.put(slotReference, cache);
}
return new Statistics(rowCount, columnStatisticMap);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 00fab11d31..848161bb31 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -39,8 +39,8 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.StringSubstitutor;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.text.SimpleDateFormat;
import java.util.Date;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java
index 3e2a7bf1c5..9ab062f541 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskScheduler.java
@@ -17,15 +17,12 @@
package org.apache.doris.statistics;
-import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.datasource.CatalogIf;
-import org.apache.doris.statistics.AnalysisTaskInfo.JobType;
+import org.apache.doris.statistics.util.StatisticsUtil;
-import com.google.common.base.Preconditions;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.util.Comparator;
import java.util.HashSet;
@@ -47,27 +44,28 @@ public class AnalysisTaskScheduler {
private final Set<BaseAnalysisTask> manualJobSet = new HashSet<>();
- public synchronized void schedule(AnalysisTaskInfo analysisJobInfo) {
- CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(analysisJobInfo.catalogName);
- Preconditions.checkArgument(catalog != null);
- DatabaseIf db = catalog.getDbNullable(analysisJobInfo.dbName);
- Preconditions.checkArgument(db != null);
- TableIf table = db.getTableNullable(analysisJobInfo.tblName);
- Preconditions.checkArgument(table != null);
- BaseAnalysisTask analysisTask = table.createAnalysisTask(this, analysisJobInfo);
- addToManualJobQueue(analysisTask);
- if (analysisJobInfo.jobType.equals(JobType.MANUAL)) {
- return;
+ public synchronized void schedule(AnalysisTaskInfo analysisTaskInfo) {
+ try {
+ TableIf table = StatisticsUtil.findTable(analysisTaskInfo.catalogName,
+ analysisTaskInfo.dbName, analysisTaskInfo.tblName);
+ BaseAnalysisTask analysisTask = table.createAnalysisTask(this, analysisTaskInfo);
+ switch (analysisTaskInfo.jobType) {
+ case MANUAL:
+ addToManualJobQueue(analysisTask);
+ break;
+ case SYSTEM:
+ addToSystemQueue(analysisTask);
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown job type: " + analysisTaskInfo.jobType);
+ }
+ } catch (Throwable t) {
+ Env.getCurrentEnv().getAnalysisManager().updateTaskStatus(
+ analysisTaskInfo, AnalysisState.FAILED, t.getMessage(), System.currentTimeMillis());
}
- addToSystemQueue(analysisTask);
}
- private void removeFromSystemQueue(BaseAnalysisTask analysisJobInfo) {
- if (manualJobSet.contains(analysisJobInfo)) {
- systemJobQueue.remove(analysisJobInfo);
- manualJobSet.remove(analysisJobInfo);
- }
- }
+ // Make sure invoker of this method is synchronized on object.
private void addToSystemQueue(BaseAnalysisTask analysisJobInfo) {
if (systemJobSet.contains(analysisJobInfo)) {
@@ -78,6 +76,7 @@ public class AnalysisTaskScheduler {
notify();
}
+ // Make sure invoker of this method is synchronized on object.
private void addToManualJobQueue(BaseAnalysisTask analysisJobInfo) {
if (manualJobSet.contains(analysisJobInfo)) {
return;
@@ -90,10 +89,10 @@ public class AnalysisTaskScheduler {
public synchronized BaseAnalysisTask getPendingTasks() {
while (true) {
if (!manualJobQueue.isEmpty()) {
- return manualJobQueue.poll();
+ return pollAndRemove(manualJobQueue, manualJobSet);
}
if (!systemJobQueue.isEmpty()) {
- return systemJobQueue.poll();
+ return pollAndRemove(systemJobQueue, systemJobSet);
}
try {
wait();
@@ -103,4 +102,11 @@ public class AnalysisTaskScheduler {
}
}
}
+
+ // Poll from queue, remove from set. Make sure invoker of this method is synchronized on object.
+ private BaseAnalysisTask pollAndRemove(Queue<BaseAnalysisTask> q, Set<BaseAnalysisTask> s) {
+ BaseAnalysisTask t = q.poll();
+ s.remove(t);
+ return t;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index 54793881ab..ab518d66bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -20,6 +20,7 @@ package org.apache.doris.statistics;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.qe.StmtExecutor;
@@ -29,10 +30,18 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.HashSet;
+import java.util.Set;
+
public abstract class BaseAnalysisTask {
public static final Logger LOG = LogManager.getLogger(BaseAnalysisTask.class);
+ /**
+ * Stats stored in the column_statistics table basically has two types, `part_id` is null which means it is
+ * aggregate from partition level stats, `part_id` is not null which means it is partition level stats.
+ * For latter, it's id field contains part id, for previous doesn't.
+ */
protected static final String INSERT_PART_STATISTICS = "INSERT INTO "
+ "${internalDB}.${columnStatTbl}"
+ " SELECT "
@@ -93,6 +102,8 @@ public abstract class BaseAnalysisTask {
protected AnalysisState analysisState;
+ protected Set<PrimitiveType> unsupportedType = new HashSet<>();
+
@VisibleForTesting
public BaseAnalysisTask() {
@@ -104,7 +115,17 @@ public abstract class BaseAnalysisTask {
init(info);
}
+ protected void initUnsupportedType() {
+ unsupportedType.add(PrimitiveType.HLL);
+ unsupportedType.add(PrimitiveType.BITMAP);
+ unsupportedType.add(PrimitiveType.ARRAY);
+ unsupportedType.add(PrimitiveType.MAP);
+ unsupportedType.add(PrimitiveType.JSONB);
+ unsupportedType.add(PrimitiveType.STRUCT);
+ }
+
private void init(AnalysisTaskInfo info) {
+ initUnsupportedType();
catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(info.catalogName);
if (catalog == null) {
Env.getCurrentEnv().getAnalysisManager().updateTaskStatus(info, AnalysisState.FAILED,
@@ -127,9 +148,11 @@ public abstract class BaseAnalysisTask {
|| info.analysisType.equals(AnalysisType.HISTOGRAM))) {
col = tbl.getColumn(info.colName);
if (col == null) {
- Env.getCurrentEnv().getAnalysisManager().updateTaskStatus(
- info, AnalysisState.FAILED, String.format("Column with name %s not exists", info.tblName),
- System.currentTimeMillis());
+ throw new RuntimeException(String.format("Column with name %s not exists", info.tblName));
+ }
+ if (isUnsupportedType(col.getType().getPrimitiveType())) {
+ throw new RuntimeException(String.format("Column with type %s is not supported",
+ col.getType().toString()));
}
}
@@ -165,4 +188,8 @@ public abstract class BaseAnalysisTask {
return "COUNT(1) * " + column.getType().getSlotSize();
}
+ private boolean isUnsupportedType(PrimitiveType type) {
+ return unsupportedType.contains(type);
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnLevelStatisticCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnLevelStatisticCache.java
deleted file mode 100644
index a01eb4678e..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnLevelStatisticCache.java
+++ /dev/null
@@ -1,52 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.statistics;
-
-public class ColumnLevelStatisticCache {
-
- public Histogram histogram;
-
- public ColumnStatistic columnStatistic;
-
- public ColumnLevelStatisticCache() {
- }
-
- public ColumnLevelStatisticCache(Histogram histogram, ColumnStatistic columnStatistic) {
- this.histogram = histogram;
- this.columnStatistic = columnStatistic;
- }
-
- public Histogram getHistogram() {
- return histogram;
- }
-
- public void setHistogram(Histogram histogram) {
- this.histogram = histogram;
- }
-
- public ColumnStatistic getColumnStatistic() {
- if (columnStatistic != null) {
- return columnStatistic;
- }
- return ColumnStatistic.UNKNOWN;
- }
-
- public void setColumnStatistic(ColumnStatistic columnStatistic) {
- this.columnStatistic = columnStatistic;
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
index eeb2b5dcac..5b58f5644a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
@@ -94,6 +94,7 @@ public class ColumnStatistic {
public final LiteralExpr minExpr;
public final LiteralExpr maxExpr;
+ // assign value when do stats estimation.
public final Histogram histogram;
public ColumnStatistic(double count, double ndv, double originalNdv, double avgSizeByte,
@@ -154,7 +155,8 @@ public class ColumnStatistic {
}
columnStatisticBuilder.setSelectivity(1.0);
columnStatisticBuilder.setOriginalNdv(ndv);
- Histogram histogram = Env.getCurrentEnv().getStatisticsCache().getHistogram(tblId, idxId, colName);
+ Histogram histogram = Env.getCurrentEnv().getStatisticsCache().getHistogram(tblId, idxId, colName)
+ .orElse(null);
columnStatisticBuilder.setHistogram(histogram);
return columnStatisticBuilder.build();
} catch (Exception e) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
new file mode 100644
index 0000000000..c6ee9103cc
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.common.FeConstants;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
+import org.apache.doris.statistics.util.StatisticsUtil;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.text.StringSubstitutor;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CompletionException;
+
+public class ColumnStatisticsCacheLoader extends StatisticsCacheLoader<Optional<ColumnStatistic>> {
+
+ private static final Logger LOG = LogManager.getLogger(ColumnStatisticsCacheLoader.class);
+
+ private static final String QUERY_COLUMN_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME
+ + "." + StatisticConstants.STATISTIC_TBL_NAME + " WHERE "
+ + "id = CONCAT('${tblId}', '-', ${idxId}, '-', '${colId}')";
+
+ @Override
+ protected Optional<ColumnStatistic> doLoad(StatisticsCacheKey key) {
+ Map<String, String> params = new HashMap<>();
+ params.put("tblId", String.valueOf(key.tableId));
+ params.put("idxId", String.valueOf(key.idxId));
+ params.put("colId", String.valueOf(key.colName));
+
+ List<ColumnStatistic> columnStatistics;
+ List<ResultRow> columnResult =
+ StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
+ .replace(QUERY_COLUMN_STATISTICS));
+ try {
+ columnStatistics = StatisticsUtil.deserializeToColumnStatistics(columnResult);
+ } catch (Exception e) {
+ LOG.warn("Failed to deserialize column statistics", e);
+ throw new CompletionException(e);
+ }
+ if (CollectionUtils.isEmpty(columnStatistics)) {
+ return Optional.empty();
+ } else {
+ return Optional.of(columnStatistics.get(0));
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramCacheLoader.java
new file mode 100644
index 0000000000..0e07524092
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramCacheLoader.java
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.common.FeConstants;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
+import org.apache.doris.statistics.util.StatisticsUtil;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.text.StringSubstitutor;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CompletionException;
+
+public class HistogramCacheLoader extends StatisticsCacheLoader<Optional<Histogram>> {
+
+ private static final Logger LOG = LogManager.getLogger(HistogramCacheLoader.class);
+
+ private static final String QUERY_HISTOGRAM_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME
+ + "." + StatisticConstants.HISTOGRAM_TBL_NAME + " WHERE "
+ + "id = CONCAT('${tblId}', '-', ${idxId}, '-', '${colId}')";
+
+ @Override
+ protected Optional<Histogram> doLoad(StatisticsCacheKey key) {
+ List<Histogram> histogramStatistics;
+ Map<String, String> params = new HashMap<>();
+ params.put("tblId", String.valueOf(key.tableId));
+ params.put("idxId", String.valueOf(key.idxId));
+ params.put("colId", String.valueOf(key.colName));
+
+ List<ResultRow> histogramResult =
+ StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
+ .replace(QUERY_HISTOGRAM_STATISTICS));
+ try {
+ histogramStatistics = StatisticsUtil.deserializeToHistogramStatistics(histogramResult);
+ } catch (Exception e) {
+ LOG.warn("Failed to deserialize histogram statistics", e);
+ throw new CompletionException(e);
+ }
+ if (!CollectionUtils.isEmpty(histogramStatistics)) {
+ return Optional.of(histogramStatistics.get(0));
+ }
+ return Optional.empty();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
index 5097a86767..f5fcd11ff9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
@@ -109,6 +109,6 @@ public class HistogramTask extends BaseAnalysisTask {
this.stmtExecutor.execute();
}
- Env.getCurrentEnv().getStatisticsCache().refreshSync(tbl.getId(), -1, col.getName());
+ Env.getCurrentEnv().getStatisticsCache().refreshHistogramSync(tbl.getId(), -1, col.getName());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java
index 200cd8559e..308e46c036 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java
@@ -142,7 +142,7 @@ public class HiveAnalysisTask extends HMSAnalysisTask {
this.stmtExecutor.execute();
}
}
- Env.getCurrentEnv().getStatisticsCache().refreshSync(tbl.getId(), -1, col.getName());
+ Env.getCurrentEnv().getStatisticsCache().refreshColStatsSync(tbl.getId(), -1, col.getName());
}
private void getStatData(ColumnStatisticsData data, Map<String, String> params) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java
index c91398a0a6..1194919606 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java
@@ -124,7 +124,7 @@ public class MVAnalysisTask extends BaseAnalysisTask {
params.put("type", column.getType().toString());
StatisticsUtil.execUpdate(ANALYZE_MV_COL, params);
Env.getCurrentEnv().getStatisticsCache()
- .refreshSync(meta.getIndexId(), meta.getIndexId(), column.getName());
+ .refreshColStatsSync(meta.getIndexId(), meta.getIndexId(), column.getName());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 2509143c75..74f1d3cbad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -94,7 +94,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
String sql = stringSubstitutor.replace(ANALYZE_COLUMN_SQL_TEMPLATE);
execSQL(sql);
- Env.getCurrentEnv().getStatisticsCache().refreshSync(tbl.getId(), -1, col.getName());
+ Env.getCurrentEnv().getStatisticsCache().refreshColStatsSync(tbl.getId(), -1, col.getName());
}
@VisibleForTesting
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
index 0345c7930e..7feff48e26 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
@@ -59,9 +59,8 @@ public class StatisticConstants {
*/
public static final long STATISTICS_TASKS_TIMEOUT_IN_MS = TimeUnit.MINUTES.toMillis(10);
+ public static final long PRELOAD_RETRY_TIMES = 5;
- public static final int LOAD_TASK_LIMITS = 10;
-
- public static final double DEFAULT_INNER_JOIN_FACTOR = 0.1;
+ public static final long PRELOAD_RETRY_INTERVAL_IN_SECONDS = TimeUnit.SECONDS.toMillis(10);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
index 9add3c1bc3..6ee30bb90a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
@@ -17,18 +17,21 @@
package org.apache.doris.statistics;
-//import org.apache.doris.common.ThreadPoolManager;
-
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import com.github.benmanes.caffeine.cache.Caffeine;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -44,85 +47,159 @@ public class StatisticsCache {
= ThreadPoolManager.newDaemonFixedThreadPool(
10, Integer.MAX_VALUE, "STATS_FETCH", true);
- private final StatisticsCacheLoader cacheLoader = new StatisticsCacheLoader();
-
- private final AsyncLoadingCache<StatisticsCacheKey, ColumnLevelStatisticCache> cache = Caffeine.newBuilder()
- .maximumSize(StatisticConstants.STATISTICS_RECORDS_CACHE_SIZE)
- .expireAfterAccess(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_VALID_DURATION_IN_HOURS))
- .refreshAfterWrite(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_REFRESH_INTERVAL))
- .executor(threadPool)
- .buildAsync(cacheLoader);
+ private final ColumnStatisticsCacheLoader columnStatisticsCacheLoader = new ColumnStatisticsCacheLoader();
+ private final HistogramCacheLoader histogramCacheLoader = new HistogramCacheLoader();
+
+ private final AsyncLoadingCache<StatisticsCacheKey, Optional<ColumnStatistic>> columnStatisticsCache =
+ Caffeine.newBuilder()
+ .maximumSize(StatisticConstants.STATISTICS_RECORDS_CACHE_SIZE)
+ .expireAfterAccess(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_VALID_DURATION_IN_HOURS))
+ .refreshAfterWrite(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_REFRESH_INTERVAL))
+ .executor(threadPool)
+ .buildAsync(columnStatisticsCacheLoader);
+
+ private final AsyncLoadingCache<StatisticsCacheKey, Optional<Histogram>> histogramCache =
+ Caffeine.newBuilder()
+ .maximumSize(StatisticConstants.STATISTICS_RECORDS_CACHE_SIZE)
+ .expireAfterAccess(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_VALID_DURATION_IN_HOURS))
+ .refreshAfterWrite(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_REFRESH_INTERVAL))
+ .executor(threadPool)
+ .buildAsync(histogramCacheLoader);
{
threadPool.submit(() -> {
while (true) {
try {
- cacheLoader.removeExpiredInProgressing();
- Thread.sleep(TimeUnit.MINUTES.toMillis(15));
+ columnStatisticsCacheLoader.removeExpiredInProgressing();
+ histogramCacheLoader.removeExpiredInProgressing();
} catch (Throwable t) {
// IGNORE
}
+ Thread.sleep(TimeUnit.MINUTES.toMillis(15));
}
});
}
public ColumnStatistic getColumnStatistics(long tblId, String colName) {
- ColumnLevelStatisticCache columnLevelStatisticCache = getColumnStatistics(tblId, -1, colName);
- if (columnLevelStatisticCache == null) {
- return ColumnStatistic.UNKNOWN;
- }
- return columnLevelStatisticCache.columnStatistic;
+ return getColumnStatistics(tblId, -1, colName).orElse(ColumnStatistic.UNKNOWN);
}
- public ColumnLevelStatisticCache getColumnStatistics(long tblId, long idxId, String colName) {
+ public Optional<ColumnStatistic> getColumnStatistics(long tblId, long idxId, String colName) {
ConnectContext ctx = ConnectContext.get();
if (ctx != null && ctx.getSessionVariable().internalSession) {
- return null;
+ return Optional.empty();
}
StatisticsCacheKey k = new StatisticsCacheKey(tblId, idxId, colName);
try {
- CompletableFuture<ColumnLevelStatisticCache> f = cache.get(k);
+ CompletableFuture<Optional<ColumnStatistic>> f = columnStatisticsCache.get(k);
if (f.isDone() && f.get() != null) {
return f.get();
}
} catch (Exception e) {
LOG.warn("Unexpected exception while returning ColumnStatistic", e);
}
- return null;
+ return Optional.empty();
}
public Histogram getHistogram(long tblId, String colName) {
- return getHistogram(tblId, -1, colName);
+ return getHistogram(tblId, -1, colName).orElse(null);
}
- public Histogram getHistogram(long tblId, long idxId, String colName) {
+ public Optional<Histogram> getHistogram(long tblId, long idxId, String colName) {
ConnectContext ctx = ConnectContext.get();
if (ctx != null && ctx.getSessionVariable().internalSession) {
- return null;
+ return Optional.empty();
}
StatisticsCacheKey k = new StatisticsCacheKey(tblId, idxId, colName);
try {
- CompletableFuture<ColumnLevelStatisticCache> f = cache.get(k);
+ CompletableFuture<Optional<Histogram>> f = histogramCache.get(k);
if (f.isDone() && f.get() != null) {
- return f.get().getHistogram();
+ return f.get();
}
} catch (Exception e) {
LOG.warn("Unexpected exception while returning Histogram", e);
}
- return null;
+ return Optional.empty();
}
// TODO: finish this method.
public void eraseExpiredCache(long tblId, long idxId, String colName) {
- cache.synchronous().invalidate(new StatisticsCacheKey(tblId, idxId, colName));
+ columnStatisticsCache.synchronous().invalidate(new StatisticsCacheKey(tblId, idxId, colName));
+ }
+
+ public void updateColStatsCache(long tblId, long idxId, String colName, ColumnStatistic statistic) {
+ columnStatisticsCache.synchronous().put(new StatisticsCacheKey(tblId, idxId, colName), Optional.of(statistic));
+ }
+
+ public void refreshColStatsSync(long tblId, long idxId, String colName) {
+ columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(tblId, idxId, colName));
+ }
+
+ public void refreshHistogramSync(long tblId, long idxId, String colName) {
+ histogramCache.synchronous().refresh(new StatisticsCacheKey(tblId, idxId, colName));
}
- public void updateCache(long tblId, long idxId, String colName, ColumnLevelStatisticCache statistic) {
- cache.synchronous().put(new StatisticsCacheKey(tblId, idxId, colName), statistic);
+ public void preHeat() {
+ threadPool.submit(this::doPreHeat);
}
- public void refreshSync(long tblId, long idxId, String colName) {
- cache.synchronous().refresh(new StatisticsCacheKey(tblId, idxId, colName));
+ private void doPreHeat() {
+ List<ResultRow> recentStatsUpdatedCols = null;
+ long retryTimes = 0;
+ while (retryTimes < StatisticConstants.PRELOAD_RETRY_TIMES) {
+ try {
+ recentStatsUpdatedCols = StatisticsRepository.fetchRecentStatsUpdatedCol();
+ break;
+ } catch (Throwable t) {
+ // IGNORE
+ }
+ retryTimes++;
+ try {
+ Thread.sleep(StatisticConstants.PRELOAD_RETRY_INTERVAL_IN_SECONDS);
+ } catch (Throwable t) {
+ // IGNORE
+ }
+ }
+
+ if (CollectionUtils.isEmpty(recentStatsUpdatedCols)) {
+ return;
+ }
+ for (ResultRow r : recentStatsUpdatedCols) {
+ try {
+ String tblId = r.getColumnValue("tbl_id");
+ String idxId = r.getColumnValue("idx_id");
+ String colId = r.getColumnValue("col_id");
+ final StatisticsCacheKey k =
+ new StatisticsCacheKey(Long.parseLong(tblId), Long.parseLong(idxId), colId);
+ final ColumnStatistic c = ColumnStatistic.fromResultRow(r);
+ CompletableFuture<Optional<ColumnStatistic>> f = new CompletableFuture<Optional<ColumnStatistic>>() {
+
+ @Override
+ public Optional<ColumnStatistic> get() throws InterruptedException, ExecutionException {
+ return Optional.of(c);
+ }
+
+ @Override
+ public boolean isDone() {
+ return true;
+ }
+
+ @Override
+ public boolean complete(Optional<ColumnStatistic> value) {
+ return true;
+ }
+
+ @Override
+ public Optional<ColumnStatistic> join() {
+ return Optional.of(c);
+ }
+ };
+ columnStatisticsCache.put(k, f);
+ } catch (Throwable t) {
+ LOG.warn("Error when preheating stats cache", t);
+ }
+ }
}
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java
index 9b48876bc2..e31b545245 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheKey.java
@@ -18,6 +18,7 @@
package org.apache.doris.statistics;
import java.util.Objects;
+import java.util.StringJoiner;
public class StatisticsCacheKey {
@@ -28,6 +29,8 @@ public class StatisticsCacheKey {
public final long idxId;
public final String colName;
+ private static final String DELIMITER = "-";
+
public StatisticsCacheKey(long tableId, String colName) {
this(tableId, -1, colName);
}
@@ -54,4 +57,13 @@ public class StatisticsCacheKey {
StatisticsCacheKey k = (StatisticsCacheKey) obj;
return this.tableId == k.tableId && this.idxId == k.idxId && this.colName.equals(k.colName);
}
+
+ @Override
+ public String toString() {
+ StringJoiner sj = new StringJoiner(DELIMITER);
+ sj.add(String.valueOf(tableId));
+ sj.add(String.valueOf(idxId));
+ sj.add(colName);
+ return sj.toString();
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java
index 3417356b78..c212851a28 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCacheLoader.java
@@ -17,90 +17,35 @@
package org.apache.doris.statistics;
-import org.apache.doris.common.FeConstants;
-import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
-import org.apache.doris.statistics.util.StatisticsUtil;
-
import com.github.benmanes.caffeine.cache.AsyncCacheLoader;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.text.StringSubstitutor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.checkerframework.checker.nullness.qual.NonNull;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
-public class StatisticsCacheLoader implements AsyncCacheLoader<StatisticsCacheKey, ColumnLevelStatisticCache> {
+public abstract class StatisticsCacheLoader<V> implements AsyncCacheLoader<StatisticsCacheKey, V> {
private static final Logger LOG = LogManager.getLogger(StatisticsCacheLoader.class);
- private static final String QUERY_COLUMN_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME
- + "." + StatisticConstants.STATISTIC_TBL_NAME + " WHERE "
- + "id = CONCAT('${tblId}', '-', ${idxId}, '-', '${colId}')";
-
- private static final String QUERY_HISTOGRAM_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME
- + "." + StatisticConstants.HISTOGRAM_TBL_NAME + " WHERE "
- + "id = CONCAT('${tblId}', '-', ${idxId}, '-', '${colId}')";
-
- // TODO: Maybe we should trigger a analyze job when the required ColumnStatistic doesn't exists.
-
- private final Map<StatisticsCacheKey, CompletableFutureWithCreateTime>
- inProgressing = new HashMap<>();
+ private final Map<StatisticsCacheKey, CompletableFutureWithCreateTime<V>> inProgressing = new HashMap<>();
@Override
- public @NonNull CompletableFuture<ColumnLevelStatisticCache> asyncLoad(@NonNull StatisticsCacheKey key,
+ public @NonNull CompletableFuture<V> asyncLoad(
+ @NonNull StatisticsCacheKey key,
@NonNull Executor executor) {
- CompletableFutureWithCreateTime cfWrapper = inProgressing.get(key);
+ CompletableFutureWithCreateTime<V> cfWrapper = inProgressing.get(key);
if (cfWrapper != null) {
return cfWrapper.cf;
}
- CompletableFuture<ColumnLevelStatisticCache> future = CompletableFuture.supplyAsync(() -> {
+ CompletableFuture<V> future = CompletableFuture.supplyAsync(() -> {
long startTime = System.currentTimeMillis();
try {
- LOG.info("Query BE for column stats:{}-{} start time:{}", key.tableId, key.colName,
- startTime);
- ColumnLevelStatisticCache statistic = new ColumnLevelStatisticCache();
- Map<String, String> params = new HashMap<>();
- params.put("tblId", String.valueOf(key.tableId));
- params.put("idxId", String.valueOf(key.idxId));
- params.put("colId", String.valueOf(key.colName));
-
- List<ColumnStatistic> columnStatistics;
- List<ResultRow> columnResult =
- StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
- .replace(QUERY_COLUMN_STATISTICS));
- try {
- columnStatistics = StatisticsUtil.deserializeToColumnStatistics(columnResult);
- } catch (Exception e) {
- LOG.warn("Failed to deserialize column statistics", e);
- throw new CompletionException(e);
- }
- if (CollectionUtils.isEmpty(columnStatistics)) {
- statistic.setColumnStatistic(ColumnStatistic.UNKNOWN);
- } else {
- statistic.setColumnStatistic(columnStatistics.get(0));
- }
-
- List<Histogram> histogramStatistics;
- List<ResultRow> histogramResult =
- StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
- .replace(QUERY_HISTOGRAM_STATISTICS));
- try {
- histogramStatistics = StatisticsUtil.deserializeToHistogramStatistics(histogramResult);
- } catch (Exception e) {
- LOG.warn("Failed to deserialize histogram statistics", e);
- throw new CompletionException(e);
- }
- if (!CollectionUtils.isEmpty(histogramStatistics)) {
- statistic.setHistogram(histogramStatistics.get(0));
- }
- return statistic;
+ return doLoad(key);
} finally {
long endTime = System.currentTimeMillis();
LOG.info("Query BE for column stats:{}-{} end time:{} cost time:{}", key.tableId, key.colName,
@@ -108,11 +53,30 @@ public class StatisticsCacheLoader implements AsyncCacheLoader<StatisticsCacheKe
removeFromIProgressing(key);
}
}, executor);
- putIntoIProgressing(key, new CompletableFutureWithCreateTime(System.currentTimeMillis(), future));
+ putIntoIProgressing(key,
+ new CompletableFutureWithCreateTime<V>(System.currentTimeMillis(), future));
return future;
}
- private void putIntoIProgressing(StatisticsCacheKey k, CompletableFutureWithCreateTime v) {
+ protected abstract V doLoad(StatisticsCacheKey k);
+
+ private static class CompletableFutureWithCreateTime<V> extends CompletableFuture<V> {
+
+ public final long startTime;
+ public final CompletableFuture<V> cf;
+ private final long expiredTimeMilli = TimeUnit.MINUTES.toMillis(30);
+
+ public CompletableFutureWithCreateTime(long startTime, CompletableFuture<V> cf) {
+ this.startTime = startTime;
+ this.cf = cf;
+ }
+
+ public boolean isExpired() {
+ return System.currentTimeMillis() - startTime > expiredTimeMilli;
+ }
+ }
+
+ private void putIntoIProgressing(StatisticsCacheKey k, CompletableFutureWithCreateTime<V> v) {
synchronized (inProgressing) {
inProgressing.put(k, v);
}
@@ -131,24 +95,4 @@ public class StatisticsCacheLoader implements AsyncCacheLoader<StatisticsCacheKe
inProgressing.entrySet().removeIf(e -> e.getValue().isExpired());
}
}
-
- /**
- * To make sure any item in the inProgressing would finally be removed to avoid potential mem leak.
- */
- private static class CompletableFutureWithCreateTime extends CompletableFuture<ColumnLevelStatisticCache> {
-
- private static final long EXPIRED_TIME_MILLI = TimeUnit.MINUTES.toMillis(30);
-
- public final long startTime;
- public final CompletableFuture<ColumnLevelStatisticCache> cf;
-
- public CompletableFutureWithCreateTime(long startTime, CompletableFuture<ColumnLevelStatisticCache> cf) {
- this.startTime = startTime;
- this.cf = cf;
- }
-
- public boolean isExpired() {
- return System.currentTimeMillis() - startTime > EXPIRED_TIME_MILLI;
- }
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
index 8a5dab5b26..9a735cf98a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -83,6 +83,13 @@ public class StatisticsRepository {
private static final String DROP_TABLE_HISTOGRAM_TEMPLATE = "DELETE FROM " + FeConstants.INTERNAL_DB_NAME
+ "." + StatisticConstants.HISTOGRAM_TBL_NAME + " WHERE ${condition}";
+ private static final String FETCH_RECENT_STATS_UPDATED_COL =
+ "SELECT * FROM "
+ + FeConstants.INTERNAL_DB_NAME + "." + StatisticConstants.STATISTIC_TBL_NAME
+ + " WHERE part_id is NULL "
+ + " ORDER BY update_time DESC LIMIT "
+ + StatisticConstants.STATISTICS_RECORDS_CACHE_SIZE;
+
public static ColumnStatistic queryColumnStatisticsByName(long tableId, String colName) {
ResultRow resultRow = queryColumnStatisticById(tableId, colName);
if (resultRow == null) {
@@ -103,8 +110,8 @@ public class StatisticsRepository {
partitionIds.add(partition.getId());
}
return queryPartitionStatistics(dbObjects.table.getId(),
- colName, partitionIds).stream().map(ColumnStatistic::fromResultRow).collect(
- Collectors.toList());
+ colName, partitionIds).stream().map(ColumnStatistic::fromResultRow).collect(
+ Collectors.toList());
}
public static ResultRow queryColumnStatisticById(long tblId, String colName) {
@@ -259,16 +266,8 @@ public class StatisticsRepository {
params.put("max", max == null ? "NULL" : max);
params.put("dataSize", String.valueOf(columnStatistic.dataSize));
StatisticsUtil.execUpdate(INSERT_INTO_COLUMN_STATISTICS, params);
-
- Histogram histogram = Env.getCurrentEnv().getStatisticsCache()
- .getHistogram(objects.table.getId(), -1, colName);
-
- ColumnLevelStatisticCache statistic = new ColumnLevelStatisticCache();
- statistic.setHistogram(histogram);
- statistic.setColumnStatistic(builder.build());
-
Env.getCurrentEnv().getStatisticsCache()
- .updateCache(objects.table.getId(), -1, colName, statistic);
+ .updateColStatsCache(objects.table.getId(), -1, colName, builder.build());
}
public static void dropTableStatistics(DropTableStatsStmt dropTableStatsStmt) {
@@ -279,4 +278,8 @@ public class StatisticsRepository {
dropHistogram(dbId, tbIds, cols, partIds);
dropStatistics(dbId, tbIds, cols, partIds);
}
+
+ public static List<ResultRow> fetchRecentStatsUpdatedCol() {
+ return StatisticsUtil.execStatisticQuery(FETCH_RECENT_STATS_UPDATED_COL);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index 2c2de6e3d8..19bcb6f2c4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -284,6 +284,23 @@ public class StatisticsUtil {
return tblIf.getColumn(columnName);
}
+ @SuppressWarnings({"unchecked"})
+ public static Column findColumn(String catalogName, String dbName, String tblName, String columnName)
+ throws Throwable {
+ TableIf tableIf = findTable(catalogName, dbName, tblName);
+ return tableIf.getColumn(columnName);
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public static TableIf findTable(String catalogName, String dbName, String tblName) throws Throwable {
+ CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr()
+ .getCatalogOrException(catalogName, c -> new RuntimeException("Catalog: " + c + " not exists"));
+ DatabaseIf db = catalog.getDbOrException(dbName,
+ d -> new RuntimeException("DB: " + d + " not exists"));
+ return db.getTableOrException(tblName,
+ t -> new RuntimeException("Table: " + t + " not exists"));
+ }
+
public static boolean isNullOrEmpty(String str) {
return Optional.ofNullable(str)
.map(String::trim)
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
index ea5ce02a96..a904197362 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
@@ -17,7 +17,6 @@
package org.apache.doris.nereids.jobs.cascades;
-import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.jobs.JobContext;
@@ -53,8 +52,6 @@ public class DeriveStatsJobTest {
@Mocked
ConnectContext context;
- @Mocked
- Env env;
SlotReference slot1;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/TPCHStats.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/TPCHStats.java
deleted file mode 100644
index 7ce4e53694..0000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/TPCHStats.java
+++ /dev/null
@@ -1,2094 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.stats;
-
-import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.Env;
-import org.apache.doris.catalog.ScalarType;
-import org.apache.doris.nereids.datasets.tpch.TPCHUtils;
-import org.apache.doris.planner.PlanNodeId;
-import org.apache.doris.statistics.ColumnLevelStatisticCache;
-import org.apache.doris.statistics.ColumnStatistic;
-import org.apache.doris.statistics.ColumnStatisticBuilder;
-import org.apache.doris.statistics.Histogram;
-import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
-import org.apache.doris.statistics.util.StatisticsUtil;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.jupiter.api.Test;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-// CHECKSTYLE OFF
-public class TPCHStats extends TestStats {
-
- @Test
- public void testTPCHStats() throws Exception {
- run();
- }
-
- @Override
- protected void initEnv() throws Exception {
- createDatabase("tpch");
- connectContext.setDatabase("default_cluster:tpch");
- TPCHUtils.createTables(this);
- }
-
- @Override
- protected void initQError() {
- avgQError = 385.5745449282787;
- }
-
- @Override
- protected void initColNameToType() {
-
- colType.put("c_custkey", ScalarType.INT);
-
- colType.put("c_name", ScalarType.VARCHAR);
-
- colType.put("c_address", ScalarType.VARCHAR);
-
- colType.put("c_nationkey", ScalarType.INT);
-
- colType.put("c_phone", ScalarType.VARCHAR);
-
- colType.put("c_acctbal", ScalarType.createDecimalType(15,2));
-
- colType.put("c_mktsegment", ScalarType.VARCHAR);
-
- colType.put("c_comment", ScalarType.VARCHAR);
-
- colType.put("l_shipdate", ScalarType.DATE);
-
- colType.put("l_orderkey", ScalarType.BIGINT);
-
- colType.put("l_linenumber", ScalarType.INT);
-
- colType.put("l_partkey", ScalarType.INT);
-
- colType.put("l_suppkey", ScalarType.INT);
-
- colType.put("l_quantity", ScalarType.createDecimalType(15,2));
-
- colType.put("l_extendedprice", ScalarType.createDecimalType(15,2));
-
- colType.put("l_discount", ScalarType.createDecimalType(15,2));
-
- colType.put("l_tax", ScalarType.createDecimalType(15,2));
-
- colType.put("l_returnflag", ScalarType.VARCHAR);
-
- colType.put("l_linestatus", ScalarType.VARCHAR);
-
- colType.put("l_commitdate", ScalarType.DATE);
-
- colType.put("l_receiptdate", ScalarType.DATE);
-
- colType.put("l_shipinstruct", ScalarType.VARCHAR);
-
- colType.put("l_shipmode", ScalarType.VARCHAR);
-
- colType.put("l_comment", ScalarType.VARCHAR);
-
- colType.put("n_nationkey", ScalarType.INT);
-
- colType.put("n_name", ScalarType.VARCHAR);
-
- colType.put("n_regionkey", ScalarType.INT);
-
- colType.put("n_comment", ScalarType.VARCHAR);
-
- colType.put("o_orderkey", ScalarType.BIGINT);
-
- colType.put("o_orderdate", ScalarType.DATE);
-
- colType.put("o_custkey", ScalarType.INT);
-
- colType.put("o_orderstatus", ScalarType.VARCHAR);
-
- colType.put("o_totalprice", ScalarType.createDecimalType(15,2));
-
- colType.put("o_orderpriority", ScalarType.VARCHAR);
-
- colType.put("o_clerk", ScalarType.VARCHAR);
-
- colType.put("o_shippriority", ScalarType.INT);
-
- colType.put("o_comment", ScalarType.VARCHAR);
-
- colType.put("p_partkey", ScalarType.INT);
-
- colType.put("p_name", ScalarType.VARCHAR);
-
- colType.put("p_mfgr", ScalarType.VARCHAR);
-
- colType.put("p_brand", ScalarType.VARCHAR);
-
- colType.put("p_type", ScalarType.VARCHAR);
-
- colType.put("p_size", ScalarType.INT);
-
- colType.put("p_container", ScalarType.VARCHAR);
-
- colType.put("p_retailprice", ScalarType.createDecimalType(15,2));
-
- colType.put("p_comment", ScalarType.VARCHAR);
-
- colType.put("ps_partkey", ScalarType.INT);
-
- colType.put("ps_suppkey", ScalarType.INT);
-
- colType.put("ps_availqty", ScalarType.INT);
-
- colType.put("ps_supplycost", ScalarType.createDecimalType(15,2));
-
- colType.put("ps_comment", ScalarType.VARCHAR);
-
- colType.put("r_regionkey", ScalarType.INT);
-
- colType.put("r_name", ScalarType.VARCHAR);
-
- colType.put("r_comment", ScalarType.VARCHAR);
-
- colType.put("s_suppkey", ScalarType.INT);
-
- colType.put("s_name", ScalarType.VARCHAR);
-
- colType.put("s_address", ScalarType.VARCHAR);
-
- colType.put("s_nationkey", ScalarType.INT);
-
- colType.put("s_phone", ScalarType.VARCHAR);
-
- colType.put("s_acctbal", ScalarType.createDecimalType(15,2));
-
- colType.put("s_comment", ScalarType.VARCHAR);
-
- }
-
- @Override
- protected void initMockedReturnedRows() {
-
- queryIdToQError.put(1, 10.666666666666666);
-
- mockedExactReturnedRows.put(1, new HashMap<>());
-
- mockedExactReturnedRows.get(1).put(new PlanNodeId(0), 5916591.0);
-
- mockedExactReturnedRows.get(1).put(new PlanNodeId(1), 64.0);
-
- mockedExactReturnedRows.get(1).put(new PlanNodeId(2), 64.0);
-
- mockedExactReturnedRows.get(1).put(new PlanNodeId(3), 4.0);
-
- mockedExactReturnedRows.get(1).put(new PlanNodeId(4), 4.0);
-
- mockedExactReturnedRows.get(1).put(new PlanNodeId(5), 4.0);
-
- queryIdToQError.put(2, 460.00000000000006);
-
- mockedExactReturnedRows.put(2, new HashMap<>());
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(0), 747.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(1), 800000.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(2), 2988.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(3), 2988.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(4), 10000.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(5), 2988.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(6), 2988.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(7), 25.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(8), 2988.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(9), 2988.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(10), 1.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(11), 642.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(12), 642.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(13), 642.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(14), 460.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(15), 100.0);
-
- mockedExactReturnedRows.get(2).put(new PlanNodeId(16), 100.0);
-
- queryIdToQError.put(3, 16.289786674344537);
-
- mockedExactReturnedRows.put(3, new HashMap<>());
-
- mockedExactReturnedRows.get(3).put(new PlanNodeId(0), 727305.0);
-
- mockedExactReturnedRows.get(3).put(new PlanNodeId(1), 3241776.0);
-
- mockedExactReturnedRows.get(3).put(new PlanNodeId(2), 151331.0);
-
- mockedExactReturnedRows.get(3).put(new PlanNodeId(3), 151331.0);
-
- mockedExactReturnedRows.get(3).put(new PlanNodeId(4), 30142.0);
-
- mockedExactReturnedRows.get(3).put(new PlanNodeId(5), 30519.0);
-
- mockedExactReturnedRows.get(3).put(new PlanNodeId(6), 30519.0);
-
- mockedExactReturnedRows.get(3).put(new PlanNodeId(9), 10.0);
-
- queryIdToQError.put(4, 7.612047232987107);
-
- mockedExactReturnedRows.put(4, new HashMap<>());
-
- mockedExactReturnedRows.get(4).put(new PlanNodeId(0), 3793296.0);
-
- mockedExactReturnedRows.get(4).put(new PlanNodeId(1), 57218.0);
-
- mockedExactReturnedRows.get(4).put(new PlanNodeId(2), 52523.0);
-
- mockedExactReturnedRows.get(4).put(new PlanNodeId(3), 52523.0);
-
- mockedExactReturnedRows.get(4).put(new PlanNodeId(4), 5.0);
-
- mockedExactReturnedRows.get(4).put(new PlanNodeId(5), 5.0);
-
- mockedExactReturnedRows.get(4).put(new PlanNodeId(6), 5.0);
-
- queryIdToQError.put(5, 14.939600829885562);
-
- mockedExactReturnedRows.put(5, new HashMap<>());
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(0), 150000.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(1), 150000.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(2), 227597.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(3), 6001215.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(4), 910519.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(5), 910519.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(6), 10000.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(7), 910519.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(8), 910519.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(9), 25.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(10), 910519.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(11), 36236.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(12), 36236.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(13), 1.0);
-
- mockedExactReturnedRows.get(5).put(new PlanNodeId(14), 7243.0);
-
- queryIdToQError.put(6, 22.704312185172565);
-
- mockedExactReturnedRows.put(6, new HashMap<>());
-
- mockedExactReturnedRows.get(6).put(new PlanNodeId(0), 114160.0);
-
- mockedExactReturnedRows.get(6).put(new PlanNodeId(1), 8.0);
-
- mockedExactReturnedRows.get(6).put(new PlanNodeId(2), 8.0);
-
- mockedExactReturnedRows.get(6).put(new PlanNodeId(3), 1.0);
-
- queryIdToQError.put(7, 16.98406908487828);
-
- mockedExactReturnedRows.put(7, new HashMap<>());
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(0), 1500000.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(1), 1828450.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(2), 1828450.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(3), 1828450.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(4), 150000.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(5), 1828450.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(6), 1828450.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(7), 10000.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(8), 1828450.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(9), 1828450.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(10), 2.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(11), 145703.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(12), 145703.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(13), 2.0);
-
- mockedExactReturnedRows.get(7).put(new PlanNodeId(14), 5924.0);
-
- queryIdToQError.put(8, 172.3936534440346);
-
- mockedExactReturnedRows.put(8, new HashMap<>());
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(0), 1.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(1), 1.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(2), 25.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(3), 25.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(4), 25.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(5), 25.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(6), 150000.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(7), 150000.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(8), 457263.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(9), 457263.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(10), 6001215.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(11), 6001215.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(12), 10000.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(13), 80000.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(14), 1451.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(15), 14510000.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(16), 43693.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(17), 13389.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(18), 13389.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(19), 13389.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(20), 13389.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(21), 2603.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(22), 2603.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(23), 2.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(24), 2.0);
-
- mockedExactReturnedRows.get(8).put(new PlanNodeId(25), 2.0);
-
- queryIdToQError.put(9, 24.523403087988246);
-
- mockedExactReturnedRows.put(9, new HashMap<>());
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(0), 800000.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(1), 800000.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(2), 6001215.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(3), 6001215.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(4), 1500000.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(5), 6001215.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(6), 6001215.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(7), 10664.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(8), 319404.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(9), 319404.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(10), 10000.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(11), 319404.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(12), 319404.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(13), 25.0);
-
- mockedExactReturnedRows.get(9).put(new PlanNodeId(14), 319404.0);
-
- queryIdToQError.put(10, 17.53057668876071);
-
- mockedExactReturnedRows.put(10, new HashMap<>());
-
- mockedExactReturnedRows.get(10).put(new PlanNodeId(0), 25.0);
-
- mockedExactReturnedRows.get(10).put(new PlanNodeId(1), 25.0);
-
- mockedExactReturnedRows.get(10).put(new PlanNodeId(2), 1478870.0);
-
- mockedExactReturnedRows.get(10).put(new PlanNodeId(3), 1478870.0);
-
- mockedExactReturnedRows.get(10).put(new PlanNodeId(4), 57069.0);
-
- mockedExactReturnedRows.get(10).put(new PlanNodeId(5), 57069.0);
-
- mockedExactReturnedRows.get(10).put(new PlanNodeId(6), 150000.0);
-
- mockedExactReturnedRows.get(10).put(new PlanNodeId(7), 57069.0);
-
- mockedExactReturnedRows.get(10).put(new PlanNodeId(8), 114705.0);
-
- mockedExactReturnedRows.get(10).put(new PlanNodeId(9), 114705.0);
-
- mockedExactReturnedRows.get(10).put(new PlanNodeId(10), 37967.0);
-
- mockedExactReturnedRows.get(10).put(new PlanNodeId(11), 320.0);
-
- mockedExactReturnedRows.get(10).put(new PlanNodeId(12), 20.0);
-
- queryIdToQError.put(11, 1.1582088481002604);
-
- mockedExactReturnedRows.put(11, new HashMap<>());
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(0), 10000.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(1), 10000.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(2), 800000.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(3), 800000.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(4), 800000.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(5), 1.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(6), 31680.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(7), 29818.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(8), 29818.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(9), 10000.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(10), 10000.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(11), 800000.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(12), 800000.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(13), 800000.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(14), 1.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(15), 31680.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(16), 1.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(17), 1.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(18), 27604.0);
-
- mockedExactReturnedRows.get(11).put(new PlanNodeId(19), 27604.0);
-
- queryIdToQError.put(12, 13.846666567293623);
-
- mockedExactReturnedRows.put(12, new HashMap<>());
-
- mockedExactReturnedRows.get(12).put(new PlanNodeId(0), 30988.0);
-
- mockedExactReturnedRows.get(12).put(new PlanNodeId(1), 1500000.0);
-
- mockedExactReturnedRows.get(12).put(new PlanNodeId(2), 30988.0);
-
- mockedExactReturnedRows.get(12).put(new PlanNodeId(3), 30988.0);
-
- mockedExactReturnedRows.get(12).put(new PlanNodeId(4), 2.0);
-
- mockedExactReturnedRows.get(12).put(new PlanNodeId(5), 2.0);
-
- mockedExactReturnedRows.get(12).put(new PlanNodeId(6), 2.0);
-
- queryIdToQError.put(13, 590.0);
-
- mockedExactReturnedRows.put(13, new HashMap<>());
-
- mockedExactReturnedRows.get(13).put(new PlanNodeId(0), 1483918.0);
-
- mockedExactReturnedRows.get(13).put(new PlanNodeId(1), 1483918.0);
-
- mockedExactReturnedRows.get(13).put(new PlanNodeId(2), 150000.0);
-
- mockedExactReturnedRows.get(13).put(new PlanNodeId(3), 1533923.0);
-
- mockedExactReturnedRows.get(13).put(new PlanNodeId(4), 150000.0);
-
- mockedExactReturnedRows.get(13).put(new PlanNodeId(5), 590.0);
-
- mockedExactReturnedRows.get(13).put(new PlanNodeId(6), 590.0);
-
- mockedExactReturnedRows.get(13).put(new PlanNodeId(7), 42.0);
-
- mockedExactReturnedRows.get(13).put(new PlanNodeId(8), 42.0);
-
- mockedExactReturnedRows.get(13).put(new PlanNodeId(9), 42.0);
-
- queryIdToQError.put(14, 37.62936505020045);
-
- mockedExactReturnedRows.put(14, new HashMap<>());
-
- mockedExactReturnedRows.get(14).put(new PlanNodeId(0), 200000.0);
-
- mockedExactReturnedRows.get(14).put(new PlanNodeId(1), 200000.0);
-
- mockedExactReturnedRows.get(14).put(new PlanNodeId(2), 75983.0);
-
- mockedExactReturnedRows.get(14).put(new PlanNodeId(3), 75983.0);
-
- mockedExactReturnedRows.get(14).put(new PlanNodeId(4), 8.0);
-
- mockedExactReturnedRows.get(14).put(new PlanNodeId(5), 8.0);
-
- mockedExactReturnedRows.get(14).put(new PlanNodeId(6), 1.0);
-
- queryIdToQError.put(15, 89.86896174525964);
-
- mockedExactReturnedRows.put(15, new HashMap<>());
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(0), 225954.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(1), 225954.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(2), 10000.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(3), 10000.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(4), 1.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(5), 1.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(6), 1.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(7), 225954.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(8), 225954.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(9), 10000.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(10), 10000.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(11), 10000.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(12), 10000.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(13), 1.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(14), 1.0);
-
- mockedExactReturnedRows.get(15).put(new PlanNodeId(15), 1.0);
-
- queryIdToQError.put(16, 1250.0);
-
- mockedExactReturnedRows.put(16, new HashMap<>());
-
- mockedExactReturnedRows.get(16).put(new PlanNodeId(0), 800000.0);
-
- mockedExactReturnedRows.get(16).put(new PlanNodeId(1), 29581.0);
-
- mockedExactReturnedRows.get(16).put(new PlanNodeId(2), 118324.0);
-
- mockedExactReturnedRows.get(16).put(new PlanNodeId(3), 118324.0);
-
- mockedExactReturnedRows.get(16).put(new PlanNodeId(4), 4.0);
-
- mockedExactReturnedRows.get(16).put(new PlanNodeId(5), 118274.0);
-
- mockedExactReturnedRows.get(16).put(new PlanNodeId(6), 118274.0);
-
- mockedExactReturnedRows.get(16).put(new PlanNodeId(7), 118250.0);
-
- mockedExactReturnedRows.get(16).put(new PlanNodeId(8), 18314.0);
-
- mockedExactReturnedRows.get(16).put(new PlanNodeId(9), 18314.0);
-
- mockedExactReturnedRows.get(16).put(new PlanNodeId(10), 18314.0);
-
- queryIdToQError.put(17, 1.1230855208874255);
-
- mockedExactReturnedRows.put(17, new HashMap<>());
-
- mockedExactReturnedRows.get(17).put(new PlanNodeId(0), 204.0);
-
- mockedExactReturnedRows.get(17).put(new PlanNodeId(1), 204.0);
-
- mockedExactReturnedRows.get(17).put(new PlanNodeId(2), 6001215.0);
-
- mockedExactReturnedRows.get(17).put(new PlanNodeId(3), 6088.0);
-
- mockedExactReturnedRows.get(17).put(new PlanNodeId(4), 6088.0);
-
- mockedExactReturnedRows.get(17).put(new PlanNodeId(5), 6088.0);
-
- mockedExactReturnedRows.get(17).put(new PlanNodeId(6), 587.0);
-
- mockedExactReturnedRows.get(17).put(new PlanNodeId(7), 587.0);
-
- mockedExactReturnedRows.get(17).put(new PlanNodeId(8), 1.0);
-
- queryIdToQError.put(18, 1052.8447368421052);
-
- mockedExactReturnedRows.put(18, new HashMap<>());
-
- mockedExactReturnedRows.get(18).put(new PlanNodeId(0), 6001215.0);
-
- mockedExactReturnedRows.get(18).put(new PlanNodeId(1), 6001215.0);
-
- mockedExactReturnedRows.get(18).put(new PlanNodeId(2), 6001215.0);
-
- mockedExactReturnedRows.get(18).put(new PlanNodeId(3), 57.0);
-
- mockedExactReturnedRows.get(18).put(new PlanNodeId(4), 1500000.0);
-
- mockedExactReturnedRows.get(18).put(new PlanNodeId(5), 57.0);
-
- mockedExactReturnedRows.get(18).put(new PlanNodeId(6), 57.0);
-
- mockedExactReturnedRows.get(18).put(new PlanNodeId(7), 150000.0);
-
- mockedExactReturnedRows.get(18).put(new PlanNodeId(8), 57.0);
-
- mockedExactReturnedRows.get(18).put(new PlanNodeId(9), 399.0);
-
- mockedExactReturnedRows.get(18).put(new PlanNodeId(10), 57.0);
-
- mockedExactReturnedRows.get(18).put(new PlanNodeId(11), 57.0);
-
- mockedExactReturnedRows.get(18).put(new PlanNodeId(12), 57.0);
-
- queryIdToQError.put(19, 4.944574565005941);
-
- mockedExactReturnedRows.put(19, new HashMap<>());
-
- mockedExactReturnedRows.get(19).put(new PlanNodeId(0), 485.0);
-
- mockedExactReturnedRows.get(19).put(new PlanNodeId(1), 485.0);
-
- mockedExactReturnedRows.get(19).put(new PlanNodeId(2), 128371.0);
-
- mockedExactReturnedRows.get(19).put(new PlanNodeId(3), 121.0);
-
- mockedExactReturnedRows.get(19).put(new PlanNodeId(4), 121.0);
-
- mockedExactReturnedRows.get(19).put(new PlanNodeId(5), 1.0);
-
- queryIdToQError.put(20, 47.94983607065881);
-
- mockedExactReturnedRows.put(20, new HashMap<>());
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(0), 1.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(1), 1.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(2), 909455.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(3), 909455.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(4), 543210.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(5), 543210.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(6), 2127.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(7), 800000.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(8), 8508.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(9), 5833.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(10), 5833.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(11), 10000.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(12), 4397.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(13), 186.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(14), 186.0);
-
- mockedExactReturnedRows.get(20).put(new PlanNodeId(15), 186.0);
-
- queryIdToQError.put(21, 24.328358395723587);
-
- mockedExactReturnedRows.put(21, new HashMap<>());
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(0), 3793296.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(1), 3793296.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(2), 1.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(3), 1.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(4), 729413.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(5), 6001215.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(6), 3793296.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(7), 3657708.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(8), 3657708.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(9), 10000.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(10), 3657708.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(11), 3657708.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(12), 1762253.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(14), 73089.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(15), 4141.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(16), 4141.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(17), 411.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(18), 411.0);
-
- mockedExactReturnedRows.get(21).put(new PlanNodeId(19), 100.0);
-
- queryIdToQError.put(22, 4604.465736181143);
-
- mockedExactReturnedRows.put(22, new HashMap<>());
-
- mockedExactReturnedRows.get(22).put(new PlanNodeId(0), 1500000.0);
-
- mockedExactReturnedRows.get(22).put(new PlanNodeId(1), 1500000.0);
-
- mockedExactReturnedRows.get(22).put(new PlanNodeId(2), 42015.0);
-
- mockedExactReturnedRows.get(22).put(new PlanNodeId(3), 14086.0);
-
- mockedExactReturnedRows.get(22).put(new PlanNodeId(4), 14086.0);
-
- mockedExactReturnedRows.get(22).put(new PlanNodeId(5), 38120.0);
-
- mockedExactReturnedRows.get(22).put(new PlanNodeId(6), 38120.0);
-
- mockedExactReturnedRows.get(22).put(new PlanNodeId(7), 1.0);
-
- mockedExactReturnedRows.get(22).put(new PlanNodeId(8), 1.0);
-
- mockedExactReturnedRows.get(22).put(new PlanNodeId(9), 6384.0);
-
- mockedExactReturnedRows.get(22).put(new PlanNodeId(10), 7.0);
-
- mockedExactReturnedRows.get(22).put(new PlanNodeId(11), 7.0);
-
- }
-
- @Override
- protected void initMockedColumnsStats() {
-
- values = new ArrayList<>();
- values.add("10313--1-l_linenumber");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_linenumber");
- values.add("None");
- values.add("6001215");
- values.add("7");
- values.add("0");
- values.add("1");
- values.add("7");
- values.add("24004860");
- values.add("2023-03-26 22:38:18");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_linenumber",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_shipdate");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_shipdate");
- values.add("None");
- values.add("6001215");
- values.add("2549");
- values.add("0");
- values.add("1992-01-02");
- values.add("1998-12-01");
- values.add("96019440");
- values.add("2023-03-26 22:38:20");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_shipdate",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10509--1-o_comment");
- values.add("0");
- values.add("10310");
- values.add("10509");
- values.add("-1");
- values.add("o_comment");
- values.add("None");
- values.add("1500000");
- values.add("1465415");
- values.add("0");
- values.add(" Tiresias about the blithely ironic a");
- values.add("zzle? furiously ironic instructions among the unusual t");
- values.add("72770808");
- values.add("2023-03-26 22:38:22");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("o_comment",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10704--1-ps_partkey");
- values.add("0");
- values.add("10310");
- values.add("10704");
- values.add("-1");
- values.add("ps_partkey");
- values.add("None");
- values.add("800000");
- values.add("196099");
- values.add("0");
- values.add("1");
- values.add("200000");
- values.add("3200000");
- values.add("2023-03-26 22:38:23");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("ps_partkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10756--1-p_name");
- values.add("0");
- values.add("10310");
- values.add("10756");
- values.add("-1");
- values.add("p_name");
- values.add("None");
- values.add("200000");
- values.add("200265");
- values.add("0");
- values.add("almond antique blue royal burnished");
- values.add("yellow white seashell lavender black");
- values.add("6550221");
- values.add("2023-03-26 22:38:22");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("p_name",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10807--1-c_phone");
- values.add("0");
- values.add("10310");
- values.add("10807");
- values.add("-1");
- values.add("c_phone");
- values.add("None");
- values.add("150000");
- values.add("152666");
- values.add("0");
- values.add("10-100-106-1617");
- values.add("34-999-618-6881");
- values.add("2250000");
- values.add("2023-03-26 22:38:16");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("c_phone",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10858--1-s_acctbal");
- values.add("0");
- values.add("10310");
- values.add("10858");
- values.add("-1");
- values.add("s_acctbal");
- values.add("None");
- values.add("10000");
- values.add("9954");
- values.add("0");
- values.add("-998.220000000");
- values.add("9999.720000000");
- values.add("160000");
- values.add("2023-03-26 22:38:25");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("s_acctbal",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_linestatus");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_linestatus");
- values.add("None");
- values.add("6001215");
- values.add("2");
- values.add("0");
- values.add("F");
- values.add("O");
- values.add("6001215");
- values.add("2023-03-26 22:38:21");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_linestatus",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_orderkey");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_orderkey");
- values.add("None");
- values.add("6001215");
- values.add("1510112");
- values.add("0");
- values.add("1");
- values.add("6000000");
- values.add("48009720");
- values.add("2023-03-26 22:38:18");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_orderkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_partkey");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_partkey");
- values.add("None");
- values.add("6001215");
- values.add("196099");
- values.add("0");
- values.add("1");
- values.add("200000");
- values.add("24004860");
- values.add("2023-03-26 22:38:18");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_partkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_shipmode");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_shipmode");
- values.add("None");
- values.add("6001215");
- values.add("7");
- values.add("0");
- values.add("AIR");
- values.add("TRUCK");
- values.add("25717034");
- values.add("2023-03-26 22:38:20");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_shipmode",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10704--1-ps_supplycost");
- values.add("0");
- values.add("10310");
- values.add("10704");
- values.add("-1");
- values.add("ps_supplycost");
- values.add("None");
- values.add("800000");
- values.add("100274");
- values.add("0");
- values.add("1.000000000");
- values.add("1000.000000000");
- values.add("12800000");
- values.add("2023-03-26 22:38:23");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("ps_supplycost",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10807--1-c_address");
- values.add("0");
- values.add("10310");
- values.add("10807");
- values.add("-1");
- values.add("c_address");
- values.add("None");
- values.add("150000");
- values.add("150133");
- values.add("0");
- values.add(" 2uZwVhQvwA");
- values.add("zzxGktzXTMKS1BxZlgQ9nqQ");
- values.add("3758056");
- values.add("2023-03-26 22:38:16");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("c_address",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10858--1-s_address");
- values.add("0");
- values.add("10310");
- values.add("10858");
- values.add("-1");
- values.add("s_address");
- values.add("None");
- values.add("10000");
- values.add("9888");
- values.add("0");
- values.add(" 9aW1wwnBJJPnCx,nox0MA48Y0zpI1IeVfYZ");
- values.add("zzfDhdtZcvmVzA8rNFU,Yctj1zBN");
- values.add("249771");
- values.add("2023-03-26 22:38:25");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("s_address",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10858--1-s_suppkey");
- values.add("0");
- values.add("10310");
- values.add("10858");
- values.add("-1");
- values.add("s_suppkey");
- values.add("None");
- values.add("10000");
- values.add("10009");
- values.add("0");
- values.add("1");
- values.add("10000");
- values.add("40000");
- values.add("2023-03-26 22:38:25");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("s_suppkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_returnflag");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_returnflag");
- values.add("None");
- values.add("6001215");
- values.add("3");
- values.add("0");
- values.add("A");
- values.add("R");
- values.add("6001215");
- values.add("2023-03-26 22:38:20");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_returnflag",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_shipinstruct");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_shipinstruct");
- values.add("None");
- values.add("6001215");
- values.add("4");
- values.add("0");
- values.add("COLLECT COD");
- values.add("TAKE BACK RETURN");
- values.add("72006409");
- values.add("2023-03-26 22:38:20");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_shipinstruct",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_tax");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_tax");
- values.add("None");
- values.add("6001215");
- values.add("9");
- values.add("0");
- values.add("0.000000000");
- values.add("0.080000000");
- values.add("96019440");
- values.add("2023-03-26 22:38:19");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_tax",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10509--1-o_clerk");
- values.add("0");
- values.add("10310");
- values.add("10509");
- values.add("-1");
- values.add("o_clerk");
- values.add("None");
- values.add("1500000");
- values.add("988");
- values.add("0");
- values.add("Clerk#000000001");
- values.add("Clerk#000001000");
- values.add("22500000");
- values.add("2023-03-26 22:38:22");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("o_clerk",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10509--1-o_shippriority");
- values.add("0");
- values.add("10310");
- values.add("10509");
- values.add("-1");
- values.add("o_shippriority");
- values.add("None");
- values.add("1500000");
- values.add("1");
- values.add("0");
- values.add("0");
- values.add("0");
- values.add("6000000");
- values.add("2023-03-26 22:38:22");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("o_shippriority",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10509--1-o_totalprice");
- values.add("0");
- values.add("10310");
- values.add("10509");
- values.add("-1");
- values.add("o_totalprice");
- values.add("None");
- values.add("1500000");
- values.add("1462416");
- values.add("0");
- values.add("857.710000000");
- values.add("555285.160000000");
- values.add("24000000");
- values.add("2023-03-26 22:38:21");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("o_totalprice",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10704--1-ps_comment");
- values.add("0");
- values.add("10310");
- values.add("10704");
- values.add("-1");
- values.add("ps_comment");
- values.add("None");
- values.add("800000");
- values.add("794782");
- values.add("0");
- values.add(
- " Tiresias according to the quiet courts sleep against the ironic, final requests. carefully unusual requests affix fluffily quickly ironic packages. regular ");
- values.add(
- "zzle. unusual decoys detect slyly blithely express frays. furiously ironic packages about the bold accounts are close requests. slowly silent reque");
- values.add("98891983");
- values.add("2023-03-26 22:38:23");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("ps_comment",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10704--1-ps_suppkey");
- values.add("0");
- values.add("10310");
- values.add("10704");
- values.add("-1");
- values.add("ps_suppkey");
- values.add("None");
- values.add("800000");
- values.add("10009");
- values.add("0");
- values.add("1");
- values.add("10000");
- values.add("3200000");
- values.add("2023-03-26 22:38:23");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("ps_suppkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10756--1-p_mfgr");
- values.add("0");
- values.add("10310");
- values.add("10756");
- values.add("-1");
- values.add("p_mfgr");
- values.add("None");
- values.add("200000");
- values.add("5");
- values.add("0");
- values.add("Manufacturer#1");
- values.add("Manufacturer#5");
- values.add("2800000");
- values.add("2023-03-26 22:38:22");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("p_mfgr",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10756--1-p_type");
- values.add("0");
- values.add("10310");
- values.add("10756");
- values.add("-1");
- values.add("p_type");
- values.add("None");
- values.add("200000");
- values.add("150");
- values.add("0");
- values.add("ECONOMY ANODIZED BRASS");
- values.add("STANDARD POLISHED TIN");
- values.add("4119946");
- values.add("2023-03-26 22:38:23");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("p_type",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10858--1-s_nationkey");
- values.add("0");
- values.add("10310");
- values.add("10858");
- values.add("-1");
- values.add("s_nationkey");
- values.add("None");
- values.add("10000");
- values.add("25");
- values.add("0");
- values.add("0");
- values.add("24");
- values.add("40000");
- values.add("2023-03-26 22:38:24");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("s_nationkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10885--1-n_nationkey");
- values.add("0");
- values.add("10310");
- values.add("10885");
- values.add("-1");
- values.add("n_nationkey");
- values.add("None");
- values.add("25");
- values.add("25");
- values.add("0");
- values.add("0");
- values.add("24");
- values.add("100");
- values.add("2023-03-26 22:38:20");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("n_nationkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10890--1-r_comment");
- values.add("0");
- values.add("10310");
- values.add("10890");
- values.add("-1");
- values.add("r_comment");
- values.add("None");
- values.add("5");
- values.add("5");
- values.add("0");
- values.add("ges. thinly even pinto beans ca");
- values.add(
- "uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl");
- values.add("330");
- values.add("2023-03-26 22:38:23");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("r_comment",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_extendedprice");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_extendedprice");
- values.add("None");
- values.add("6001215");
- values.add("929697");
- values.add("0");
- values.add("901.000000000");
- values.add("104949.500000000");
- values.add("96019440");
- values.add("2023-03-26 22:38:21");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_extendedprice",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_receiptdate");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_receiptdate");
- values.add("None");
- values.add("6001215");
- values.add("2576");
- values.add("0");
- values.add("1992-01-04");
- values.add("1998-12-31");
- values.add("96019440");
- values.add("2023-03-26 22:38:18");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_receiptdate",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10509--1-o_orderstatus");
- values.add("0");
- values.add("10310");
- values.add("10509");
- values.add("-1");
- values.add("o_orderstatus");
- values.add("None");
- values.add("1500000");
- values.add("3");
- values.add("0");
- values.add("F");
- values.add("P");
- values.add("1500000");
- values.add("2023-03-26 22:38:22");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("o_orderstatus",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10704--1-ps_availqty");
- values.add("0");
- values.add("10310");
- values.add("10704");
- values.add("-1");
- values.add("ps_availqty");
- values.add("None");
- values.add("800000");
- values.add("10008");
- values.add("0");
- values.add("1");
- values.add("9999");
- values.add("3200000");
- values.add("2023-03-26 22:38:23");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("ps_availqty",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10807--1-c_comment");
- values.add("0");
- values.add("10310");
- values.add("10807");
- values.add("-1");
- values.add("c_comment");
- values.add("None");
- values.add("150000");
- values.add("147491");
- values.add("0");
- values.add(
- " Tiresias according to the slyly blithe instructions detect quickly at the slyly express courts. express dinos wake ");
- values.add("zzle. blithely regular instructions cajol");
- values.add("10876099");
- values.add("2023-03-26 22:38:15");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("c_comment",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10807--1-c_mktsegment");
- values.add("0");
- values.add("10310");
- values.add("10807");
- values.add("-1");
- values.add("c_mktsegment");
- values.add("None");
- values.add("150000");
- values.add("5");
- values.add("0");
- values.add("AUTOMOBILE");
- values.add("MACHINERY");
- values.add("1349610");
- values.add("2023-03-26 22:38:16");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("c_mktsegment",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10890--1-r_name");
- values.add("0");
- values.add("10310");
- values.add("10890");
- values.add("-1");
- values.add("r_name");
- values.add("None");
- values.add("5");
- values.add("5");
- values.add("0");
- values.add("AFRICA");
- values.add("MIDDLE EAST");
- values.add("34");
- values.add("2023-03-26 22:38:23");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("r_name",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_discount");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_discount");
- values.add("None");
- values.add("6001215");
- values.add("11");
- values.add("0");
- values.add("0.000000000");
- values.add("0.100000000");
- values.add("96019440");
- values.add("2023-03-26 22:38:19");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_discount",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_quantity");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_quantity");
- values.add("None");
- values.add("6001215");
- values.add("50");
- values.add("0");
- values.add("1.000000000");
- values.add("50.000000000");
- values.add("96019440");
- values.add("2023-03-26 22:38:20");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_quantity",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10509--1-o_orderpriority");
- values.add("0");
- values.add("10310");
- values.add("10509");
- values.add("-1");
- values.add("o_orderpriority");
- values.add("None");
- values.add("1500000");
- values.add("5");
- values.add("0");
- values.add("1-URGENT");
- values.add("5-LOW");
- values.add("12599829");
- values.add("2023-03-26 22:38:22");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("o_orderpriority",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10756--1-p_brand");
- values.add("0");
- values.add("10310");
- values.add("10756");
- values.add("-1");
- values.add("p_brand");
- values.add("None");
- values.add("200000");
- values.add("25");
- values.add("0");
- values.add("Brand#11");
- values.add("Brand#55");
- values.add("1600000");
- values.add("2023-03-26 22:38:23");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("p_brand",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10756--1-p_size");
- values.add("0");
- values.add("10310");
- values.add("10756");
- values.add("-1");
- values.add("p_size");
- values.add("None");
- values.add("200000");
- values.add("50");
- values.add("0");
- values.add("1");
- values.add("50");
- values.add("800000");
- values.add("2023-03-26 22:38:23");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("p_size",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10890--1-r_regionkey");
- values.add("0");
- values.add("10310");
- values.add("10890");
- values.add("-1");
- values.add("r_regionkey");
- values.add("None");
- values.add("5");
- values.add("5");
- values.add("0");
- values.add("0");
- values.add("4");
- values.add("20");
- values.add("2023-03-26 22:38:23");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("r_regionkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_comment");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_comment");
- values.add("None");
- values.add("6001215");
- values.add("4619207");
- values.add("0");
- values.add(" Tiresias ");
- values.add("zzle? slyly final platelets sleep quickly. ");
- values.add("158997209");
- values.add("2023-03-26 22:38:22");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_comment",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_commitdate");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_commitdate");
- values.add("None");
- values.add("6001215");
- values.add("2485");
- values.add("0");
- values.add("1992-01-31");
- values.add("1998-10-31");
- values.add("96019440");
- values.add("2023-03-26 22:38:18");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_commitdate",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10313--1-l_suppkey");
- values.add("0");
- values.add("10310");
- values.add("10313");
- values.add("-1");
- values.add("l_suppkey");
- values.add("None");
- values.add("6001215");
- values.add("10009");
- values.add("0");
- values.add("1");
- values.add("10000");
- values.add("24004860");
- values.add("2023-03-26 22:38:19");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("l_suppkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10509--1-o_custkey");
- values.add("0");
- values.add("10310");
- values.add("10509");
- values.add("-1");
- values.add("o_custkey");
- values.add("None");
- values.add("1500000");
- values.add("99149");
- values.add("0");
- values.add("1");
- values.add("149999");
- values.add("6000000");
- values.add("2023-03-26 22:38:22");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("o_custkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10509--1-o_orderdate");
- values.add("0");
- values.add("10310");
- values.add("10509");
- values.add("-1");
- values.add("o_orderdate");
- values.add("None");
- values.add("1500000");
- values.add("2428");
- values.add("0");
- values.add("1992-01-01");
- values.add("1998-08-02");
- values.add("24000000");
- values.add("2023-03-26 22:38:22");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("o_orderdate",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10509--1-o_orderkey");
- values.add("0");
- values.add("10310");
- values.add("10509");
- values.add("-1");
- values.add("o_orderkey");
- values.add("None");
- values.add("1500000");
- values.add("1510112");
- values.add("0");
- values.add("1");
- values.add("6000000");
- values.add("12000000");
- values.add("2023-03-26 22:38:21");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("o_orderkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10756--1-p_comment");
- values.add("0");
- values.add("10310");
- values.add("10756");
- values.add("-1");
- values.add("p_comment");
- values.add("None");
- values.add("200000");
- values.add("133106");
- values.add("0");
- values.add(" Tire");
- values.add("zzle. quickly si");
- values.add("2702438");
- values.add("2023-03-26 22:38:23");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("p_comment",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10756--1-p_partkey");
- values.add("0");
- values.add("10310");
- values.add("10756");
- values.add("-1");
- values.add("p_partkey");
- values.add("None");
- values.add("200000");
- values.add("196099");
- values.add("0");
- values.add("1");
- values.add("200000");
- values.add("800000");
- values.add("2023-03-26 22:38:23");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("p_partkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10756--1-p_retailprice");
- values.add("0");
- values.add("10310");
- values.add("10756");
- values.add("-1");
- values.add("p_retailprice");
- values.add("None");
- values.add("200000");
- values.add("21096");
- values.add("0");
- values.add("901.000000000");
- values.add("2098.990000000");
- values.add("3200000");
- values.add("2023-03-26 22:38:22");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("p_retailprice",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10807--1-c_name");
- values.add("0");
- values.add("10310");
- values.add("10807");
- values.add("-1");
- values.add("c_name");
- values.add("None");
- values.add("150000");
- values.add("150431");
- values.add("0");
- values.add("Customer#000000001");
- values.add("Customer#000150000");
- values.add("2700000");
- values.add("2023-03-26 22:38:15");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("c_name",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10885--1-n_comment");
- values.add("0");
- values.add("10310");
- values.add("10885");
- values.add("-1");
- values.add("n_comment");
- values.add("None");
- values.add("25");
- values.add("25");
- values.add("0");
- values.add(" haggle. carefully final deposits detect slyly agai");
- values.add(
- "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be");
- values.add("1857");
- values.add("2023-03-26 22:38:20");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("n_comment",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10885--1-n_name");
- values.add("0");
- values.add("10310");
- values.add("10885");
- values.add("-1");
- values.add("n_name");
- values.add("None");
- values.add("25");
- values.add("25");
- values.add("0");
- values.add("ALGERIA");
- values.add("VIETNAM");
- values.add("177");
- values.add("2023-03-26 22:38:20");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("n_name",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10885--1-n_regionkey");
- values.add("0");
- values.add("10310");
- values.add("10885");
- values.add("-1");
- values.add("n_regionkey");
- values.add("None");
- values.add("25");
- values.add("5");
- values.add("0");
- values.add("0");
- values.add("4");
- values.add("100");
- values.add("2023-03-26 22:38:20");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("n_regionkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10756--1-p_container");
- values.add("0");
- values.add("10310");
- values.add("10756");
- values.add("-1");
- values.add("p_container");
- values.add("None");
- values.add("200000");
- values.add("40");
- values.add("0");
- values.add("JUMBO BAG");
- values.add("WRAP PKG");
- values.add("1514971");
- values.add("2023-03-26 22:38:22");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("p_container",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10807--1-c_acctbal");
- values.add("0");
- values.add("10310");
- values.add("10807");
- values.add("-1");
- values.add("c_acctbal");
- values.add("None");
- values.add("150000");
- values.add("142496");
- values.add("0");
- values.add("-999.990000000");
- values.add("9999.990000000");
- values.add("2400000");
- values.add("2023-03-26 22:38:16");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("c_acctbal",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10807--1-c_custkey");
- values.add("0");
- values.add("10310");
- values.add("10807");
- values.add("-1");
- values.add("c_custkey");
- values.add("None");
- values.add("150000");
- values.add("149087");
- values.add("0");
- values.add("1");
- values.add("150000");
- values.add("600000");
- values.add("2023-03-26 22:38:16");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("c_custkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10807--1-c_nationkey");
- values.add("0");
- values.add("10310");
- values.add("10807");
- values.add("-1");
- values.add("c_nationkey");
- values.add("None");
- values.add("150000");
- values.add("25");
- values.add("0");
- values.add("0");
- values.add("24");
- values.add("600000");
- values.add("2023-03-26 22:38:16");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("c_nationkey",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10858--1-s_comment");
- values.add("0");
- values.add("10310");
- values.add("10858");
- values.add("-1");
- values.add("s_comment");
- values.add("None");
- values.add("10000");
- values.add("10039");
- values.add("0");
- values.add(" about the blithely express foxes. bli");
- values.add("zzle furiously. bold accounts haggle furiously ironic excuses. fur");
- values.add("625695");
- values.add("2023-03-26 22:38:24");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("s_comment",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10858--1-s_name");
- values.add("0");
- values.add("10310");
- values.add("10858");
- values.add("-1");
- values.add("s_name");
- values.add("None");
- values.add("10000");
- values.add("10002");
- values.add("0");
- values.add("Supplier#000000001");
- values.add("Supplier#000010000");
- values.add("180000");
- values.add("2023-03-26 22:38:24");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("s_name",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- values = new ArrayList<>();
- values.add("10858--1-s_phone");
- values.add("0");
- values.add("10310");
- values.add("10858");
- values.add("-1");
- values.add("s_phone");
- values.add("None");
- values.add("10000");
- values.add("10021");
- values.add("0");
- values.add("10-102-116-6785");
- values.add("34-998-900-4911");
- values.add("150000");
- values.add("2023-03-26 22:38:24");
-
- resultRow = new ResultRow(cols, types, values);
- stats.put("s_phone",
- new ColumnLevelStatisticCache(null, ColumnStatistic.fromResultRow(resultRow)));
-
- }
-
- public void t2est() {
- new MockUp<ColumnStatistic>() {
-
- @Mock
- public ColumnStatistic fromResultRow(ResultRow resultRow) {
- try {
- ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder();
- double count = Double.parseDouble(resultRow.getColumnValue("count"));
- columnStatisticBuilder.setCount(count);
- double ndv = Double.parseDouble(resultRow.getColumnValue("ndv"));
- if (0.99 * count < ndv && ndv < 1.01 * count) {
- ndv = count;
- }
- columnStatisticBuilder.setNdv(ndv);
- columnStatisticBuilder.setNumNulls(Double.parseDouble(resultRow.getColumnValue("null_count")));
- columnStatisticBuilder.setDataSize(Double
- .parseDouble(resultRow.getColumnValue("data_size_in_bytes")));
- columnStatisticBuilder.setAvgSizeByte(columnStatisticBuilder.getDataSize()
- / columnStatisticBuilder.getCount());
- long catalogId = Long.parseLong(resultRow.getColumnValue("catalog_id"));
- long idxId = Long.parseLong(resultRow.getColumnValue("idx_id"));
- long dbID = Long.parseLong(resultRow.getColumnValue("db_id"));
- long tblId = Long.parseLong(resultRow.getColumnValue("tbl_id"));
- String colName = resultRow.getColumnValue("col_id");
- Column col = new Column(colName, colType.get(colName));
- String min = resultRow.getColumnValue("min");
- String max = resultRow.getColumnValue("max");
- columnStatisticBuilder.setMinValue(StatisticsUtil.convertToDouble(col.getType(), min));
- columnStatisticBuilder.setMaxValue(StatisticsUtil.convertToDouble(col.getType(), max));
- columnStatisticBuilder.setMaxExpr(StatisticsUtil.readableValue(col.getType(), max));
- columnStatisticBuilder.setMinExpr(StatisticsUtil.readableValue(col.getType(), min));
- columnStatisticBuilder.setSelectivity(1.0);
- Histogram histogram = Env.getCurrentEnv().getStatisticsCache().getHistogram(tblId, idxId, colName);
- columnStatisticBuilder.setHistogram(histogram);
- return columnStatisticBuilder.build();
- } catch (Exception e) {
- return ColumnStatistic.UNKNOWN;
- }
- }
- };
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/TestStats.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/TestStats.java
deleted file mode 100644
index 9058d89b3e..0000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/TestStats.java
+++ /dev/null
@@ -1,151 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.stats;
-
-import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.PrimitiveType;
-import org.apache.doris.catalog.Type;
-import org.apache.doris.nereids.NereidsPlanner;
-import org.apache.doris.nereids.StatementContext;
-import org.apache.doris.nereids.datasets.tpch.TPCHUtils;
-import org.apache.doris.nereids.parser.NereidsParser;
-import org.apache.doris.planner.PlanNodeId;
-import org.apache.doris.qe.OriginStatement;
-import org.apache.doris.statistics.ColumnLevelStatisticCache;
-import org.apache.doris.statistics.StatisticsCache;
-import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
-import org.apache.doris.statistics.util.StatisticsUtil;
-import org.apache.doris.utframe.TestWithFeService;
-
-import mockit.Mock;
-import mockit.MockUp;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-// Assume that all column name is unique in the tested database
-// CHECKSTYLE OFF
-public abstract class TestStats extends TestWithFeService {
-
- protected Map<String/*colname*/, ColumnLevelStatisticCache> stats = new HashMap<>();
-
- protected List<String> cols = new ArrayList<String>() {{
- add("id");
- add("catalog_id");
- add("db_id");
- add("tbl_id");
- add("idx_id");
- add("col_id");
- add("part_id");
- add("count");
- add("ndv");
- add("null_count");
- add("min");
- add("max");
- add("data_size_in_bytes");
- add("update_time");
- }};
-
- protected List<PrimitiveType> types = new ArrayList<PrimitiveType>() {{
- add(PrimitiveType.VARCHAR);
- add(PrimitiveType.VARCHAR);
- add(PrimitiveType.VARCHAR);
- add(PrimitiveType.VARCHAR);
- add(PrimitiveType.VARCHAR);
- add(PrimitiveType.VARCHAR);
- add(PrimitiveType.VARCHAR);
- add(PrimitiveType.BIGINT);
- add(PrimitiveType.BIGINT);
- add(PrimitiveType.BIGINT);
- add(PrimitiveType.VARCHAR);
- add(PrimitiveType.VARCHAR);
- add(PrimitiveType.BIGINT);
- add(PrimitiveType.DATETIME);
- }};
-
- protected List<String> values = new ArrayList<>();
-
- protected ResultRow resultRow = null;
-
- protected final static Map<String, Type> colType = new HashMap<>();
-
- protected abstract void initMockedColumnsStats();
-
- protected abstract void initQError();
-
-
- protected abstract void initMockedReturnedRows();
-
- protected abstract void initEnv() throws Exception;
-
- protected abstract void initColNameToType();
-
- protected Map<Integer/*query id*/, Map<PlanNodeId, Double>> mockedExactReturnedRows = new HashMap<>();
- protected Map<Integer, Double> queryIdToQError = new HashMap<>();
-
- protected double avgQError;
-
-
- public void run() throws Exception {
- new MockUp<StatisticsUtil>() {
-
- @Mock
- public Column findColumn(long catalogId, long dbId, long tblId, long idxId, String columnName) {
- return new Column(columnName, colType.get(columnName));
- }
- };
- initMockedReturnedRows();
- initColNameToType();
- initMockedColumnsStats();
- new MockUp<StatisticsCache>() {
- @Mock
- public ColumnLevelStatisticCache getColumnStatistics(long tblId, long idxId, String colName) {
- return stats.get(colName);
- }
- };
-
- connectContext.getSessionVariable().setEnableNereidsPlanner(true);
- connectContext.getSessionVariable().enableFallbackToOriginalPlanner = false;
- StatsErrorEstimator statsErrorEstimator = new StatsErrorEstimator();
- connectContext.setStatsErrorEstimator(statsErrorEstimator);
- List<Double> qErrorList = new ArrayList<>();
- initEnv();
- for (int i = 0; i < TPCHUtils.SQLS.size(); i++) {
- String sql = TPCHUtils.SQLS.get(i);
- int sqlNumber = i + 1;
- NereidsPlanner nereidsPlanner = new NereidsPlanner(
- new StatementContext(connectContext, new OriginStatement(sql, 0)));
- NereidsParser nereidsParser = new NereidsParser();
- nereidsPlanner.plan(nereidsParser.parseSQL(sql).get(0));
- Map<PlanNodeId, Double> extractReturnedRows = mockedExactReturnedRows.get(sqlNumber);
- for (Entry<PlanNodeId, Double> entry : extractReturnedRows.entrySet()) {
- // statsErrorEstimator.setExactReturnedRow(entry.getKey(), entry.getValue());
- }
- qErrorList.add(statsErrorEstimator.calculateQError());
- statsErrorEstimator = new StatsErrorEstimator();
- connectContext.setStatsErrorEstimator(statsErrorEstimator);
- }
- // Assert.assertTrue(
- // qErrorList.stream()
- // .mapToDouble(Double::doubleValue).average().orElseGet(() -> Double.POSITIVE_INFINITY)
- // <= avgQError + 1);
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
index a0cf4524b4..5a92bbdfbe 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/CacheTest.java
@@ -19,10 +19,16 @@ package org.apache.doris.statistics;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.Type;
import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.utframe.TestWithFeService;
+import com.google.common.collect.Lists;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
import mockit.Expectations;
import mockit.Mock;
import mockit.MockUp;
@@ -40,7 +46,7 @@ import java.util.concurrent.Executor;
public class CacheTest extends TestWithFeService {
@Test
- public void testColumn(@Mocked StatisticsCacheLoader cacheLoader) throws Exception {
+ public void testColumn(@Mocked ColumnStatisticsCacheLoader cacheLoader) throws Exception {
new Expectations() {
{
cacheLoader.asyncLoad((StatisticsCacheKey) any, (Executor) any);
@@ -130,6 +136,46 @@ public class CacheTest extends TestWithFeService {
@Test
public void testLoadHistogram() throws Exception {
+ new MockUp<Histogram>() {
+
+ @Mock
+ public Histogram fromResultRow(ResultRow resultRow) {
+ try {
+ HistogramBuilder histogramBuilder = new HistogramBuilder();
+
+ Column col = new Column("abc", PrimitiveType.DATETIME);
+
+ Type dataType = col.getType();
+ histogramBuilder.setDataType(dataType);
+
+ double sampleRate = Double.parseDouble(resultRow.getColumnValue("sample_rate"));
+ histogramBuilder.setSampleRate(sampleRate);
+
+ String json = resultRow.getColumnValue("buckets");
+ JsonObject jsonObj = JsonParser.parseString(json).getAsJsonObject();
+
+ int bucketNum = jsonObj.get("num_buckets").getAsInt();
+ histogramBuilder.setNumBuckets(bucketNum);
+
+ List<Bucket> buckets = Lists.newArrayList();
+ JsonArray jsonArray = jsonObj.getAsJsonArray("buckets");
+ for (JsonElement element : jsonArray) {
+ try {
+ String bucketJson = element.toString();
+ buckets.add(Bucket.deserializeFromJson(dataType, bucketJson));
+ } catch (Throwable t) {
+ t.printStackTrace();
+ }
+
+ }
+ histogramBuilder.setBuckets(buckets);
+
+ return histogramBuilder.build();
+ } catch (Exception e) {
+ return null;
+ }
+ }
+ };
new MockUp<StatisticsUtil>() {
@Mock
@@ -185,7 +231,9 @@ public class CacheTest extends TestWithFeService {
};
StatisticsCache statisticsCache = new StatisticsCache();
+ statisticsCache.refreshHistogramSync(0, -1, "col");
+ Thread.sleep(10000);
Histogram histogram = statisticsCache.getHistogram(0, "col");
- Assertions.assertEquals(null, histogram);
+ Assertions.assertNotNull(histogram);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org