You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudberry.apache.org by ma...@apache.org on 2025/03/04 10:21:06 UTC
(cloudberry) 04/07: ao: Scan progress reporting for CREATE INDEX
This is an automated email from the ASF dual-hosted git repository.
maxyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry.git
commit 972000b8234f5e6f3c4fbce61893dd457ea58a7b
Author: Soumyadeep Chakraborty <so...@gmail.com>
AuthorDate: Fri Dec 30 13:06:43 2022 -0800
ao: Scan progress reporting for CREATE INDEX
This enables reporting for the table scan phase of index builds on AO
tables. This directly affects:
(1) pg_stat_progress_create_index.blocks_total
(2) pg_stat_progress_create_index.blocks_done
Sample:
Connect in utility mode to a segment, while CREATE INDEX is running on
an AO table:
postgres=# select pid, command, phase, blocks_total, blocks_done from pg_stat_progress_create_index;
pid | command | phase | blocks_total | blocks_done
--------+--------------+--------------------------------+--------------+-------------
633034 | CREATE INDEX | building index: scanning table | 54993 | 4412
(1 row)
This commit introduces:
(1) Accounting in AppendOnlyScanDescData to track the total number of
bytes read across all segment files. It is updated whenever a new block
is read. If the block is compressed, its compressed length is added
(uncompressed length otherwise).
(2) Reporting code in appendonly_index_build_range_scan() to report in
increments of BLCKSZ, the number of blocks scanned so far. The total
number of blocks to be scanned is also published up front and is
determined by summing the compressed eof values of all segfiles.
(3) Marks RelationGuessNumberOfBlocksFromSize() as inline, since it is
now being called in a much hotter code path.
Note: We currently only support reporting for non-concurrent index
builds and serial builds.
---
src/backend/access/appendonly/appendonlyam.c | 7 +++++++
src/backend/access/appendonly/appendonlyam_handler.c | 20 ++++++++++++++++----
src/backend/storage/buffer/bufmgr.c | 13 -------------
src/include/cdb/cdbappendonlyam.h | 20 ++++++++++++++++++++
src/include/storage/bufmgr.h | 13 ++++++++++++-
src/test/isolation2/isolation2_schedule | 2 ++
6 files changed, 57 insertions(+), 18 deletions(-)
diff --git a/src/backend/access/appendonly/appendonlyam.c b/src/backend/access/appendonly/appendonlyam.c
index 4718bedfab..1ea6bcdaca 100755
--- a/src/backend/access/appendonly/appendonlyam.c
+++ b/src/backend/access/appendonly/appendonlyam.c
@@ -104,6 +104,8 @@ static void appendonly_insert_finish_guts(AppendOnlyInsertDesc aoInsertDesc);
/* Hook for plugins to get control in appendonly_delete() */
appendonly_delete_hook_type appendonly_delete_hook = NULL;
+static void AppendOnlyScanDesc_UpdateTotalBytesRead(
+ AppendOnlyScanDesc scan);
/* ----------------
* initscan - scan code common to appendonly_beginscan and appendonly_rescan
@@ -1251,6 +1253,8 @@ getNextBlock(AppendOnlyScanDesc scan)
AppendOnlyExecutorReadBlock_GetContents(
&scan->executorReadBlock);
+ AppendOnlyScanDesc_UpdateTotalBytesRead(scan);
+
return true;
}
@@ -1581,6 +1585,9 @@ appendonly_beginrangescan_internal(Relation relation,
AccessShareLock,
appendOnlyMetaDataSnapshot);
}
+
+ scan->totalBytesRead = 0;
+
return scan;
}
diff --git a/src/backend/access/appendonly/appendonlyam_handler.c b/src/backend/access/appendonly/appendonlyam_handler.c
index b94e6196f2..1896c3960d 100644
--- a/src/backend/access/appendonly/appendonlyam_handler.c
+++ b/src/backend/access/appendonly/appendonlyam_handler.c
@@ -35,6 +35,7 @@
#include "catalog/storage_xlog.h"
#include "cdb/cdbappendonlyam.h"
#include "cdb/cdbvars.h"
+#include "commands/progress.h"
#include "commands/vacuum.h"
#include "commands/progress.h"
#include "executor/executor.h"
@@ -1555,7 +1556,7 @@ appendonly_index_build_range_scan(Relation heapRelation,
FileSegInfo **seginfo = NULL;
int segfile_count;
int64 total_blockcount = 0;
- AppendOnlyScanDesc hscan;
+ int64 previous_blkno = -1;
/*
* sanity checks
@@ -1704,10 +1705,22 @@ appendonly_index_build_range_scan(Relation heapRelation,
(numblocks != InvalidBlockNumber && currblockno >= (start_blockno + numblocks)))
continue;
+ /* Report scan progress, if asked to. */
if (progress)
{
- hscan = (AppendOnlyScanDesc) &aoscan->rs_base;
- pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE, hscan->rs_nblocks);
+ int64 current_blkno =
+ RelationGuessNumberOfBlocksFromSize(aoscan->totalBytesRead);
+
+ /* XXX: How can we report for builds with parallel scans? */
+ Assert(!aoscan->rs_base.rs_parallel);
+
+ /* As soon as a new block starts, report it as scanned */
+ if (current_blkno != previous_blkno)
+ {
+ pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
+ current_blkno);
+ previous_blkno = current_blkno;
+ }
}
aoTupleId = (AOTupleId *) &slot->tts_tid;
@@ -1804,7 +1817,6 @@ appendonly_index_validate_scan(Relation heapRelation,
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("index validate scan - feature not supported on appendoptimized relations")));
-
}
/* ------------------------------------------------------------------------
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index e95f4f81f9..6d0afd3435 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -3167,19 +3167,6 @@ RelationGetNumberOfBlocksInFork(Relation relation, ForkNumber forkNum)
return 0; /* keep compiler quiet */
}
-/*
- * GPDB: it is possible to need to calculate the number of blocks from the table
- * size. An example use case is when we are the dispatcher and we need to
- * acquire the number of blocks from all segments.
- *
- * Use the same calculation that RelationGetNumberOfBlocksInFork is using.
- */
-BlockNumber
-RelationGuessNumberOfBlocksFromSize(uint64 szbytes)
-{
- return (szbytes + (BLCKSZ - 1)) / BLCKSZ;
-}
-
/*
* BufferIsPermanent
* Determines whether a buffer will potentially still be around after
diff --git a/src/include/cdb/cdbappendonlyam.h b/src/include/cdb/cdbappendonlyam.h
index b0999d8df6..125c1de4d2 100644
--- a/src/include/cdb/cdbappendonlyam.h
+++ b/src/include/cdb/cdbappendonlyam.h
@@ -258,6 +258,11 @@ typedef struct AppendOnlyScanDescData
/* used in predicate pushdown */
ExprContext *aos_pushdown_econtext;
ExprState *aos_pushdown_qual;
+ /*
+ * The total number of bytes read, compressed, across all segment files, so
+ * far. This is used for scan progress reporting.
+ */
+ int64 totalBytesRead;
} AppendOnlyScanDescData;
@@ -493,5 +498,20 @@ extern bool AppendOnlyExecutorReadBlock_ScanNextTuple(AppendOnlyExecutorReadBloc
TupleTableSlot *slot);
extern void AppendOnlyExecutorReadBlock_GetContents(AppendOnlyExecutorReadBlock *executorReadBlock);
+/*
+ * Update total bytes read for the entire scan. If the block was compressed,
+ * update it with the compressed length. If the block was not compressed, update
+ * it with the uncompressed length.
+ */
+static inline void
+AppendOnlyScanDesc_UpdateTotalBytesRead(AppendOnlyScanDesc scan)
+{
+ Assert(scan->storageRead.isActive);
+
+ if (scan->storageRead.current.isCompressed)
+ scan->totalBytesRead += scan->storageRead.current.compressedLen;
+ else
+ scan->totalBytesRead += scan->storageRead.current.uncompressedLen;
+}
#endif /* CDBAPPENDONLYAM_H */
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index 7b1f98b933..5de4515bd7 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -216,7 +216,18 @@ extern void DropRelFileNodeBuffers(struct SMgrRelationData *smgr_reln, ForkNumbe
extern void DropRelFileNodesAllBuffers(struct SMgrRelationData **smgr_reln, int nnodes);
extern void DropDatabaseBuffers(Oid dbid);
-extern BlockNumber RelationGuessNumberOfBlocksFromSize(uint64 szbytes);
+/*
+ * GPDB: it is possible to need to calculate the number of blocks from the table
+ * size. An example use case is when we are the dispatcher and we need to
+ * acquire the number of blocks from all segments.
+ *
+ * Use the same calculation that RelationGetNumberOfBlocksInFork is using.
+ */
+static inline BlockNumber
+RelationGuessNumberOfBlocksFromSize(uint64 szbytes)
+{
+ return (szbytes + (BLCKSZ - 1)) / BLCKSZ;
+}
#define RelationGetNumberOfBlocks(reln) \
RelationGetNumberOfBlocksInFork(reln, MAIN_FORKNUM)
diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule
index c13f8cf750..f1ac29f0c5 100644
--- a/src/test/isolation2/isolation2_schedule
+++ b/src/test/isolation2/isolation2_schedule
@@ -234,6 +234,8 @@ test: enable_autovacuum
test: idle_gang_cleaner
# test idle_in_transaction_session_timeout
+test: ao_index_build_progress
+
# Tests for FTS
test: fts_errors
test: segwalrep/replication_keeps_crash
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cloudberry.apache.org
For additional commands, e-mail: commits-help@cloudberry.apache.org