You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudberry.apache.org by ma...@apache.org on 2025/03/05 03:55:59 UTC
(cloudberry) branch main updated: [ORCA] Support PAX AM in ORCA
This is an automated email from the ASF dual-hosted git repository.
maxyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry.git
The following commit(s) were added to refs/heads/main by this push:
new 0af03caefc [ORCA] Support PAX AM in ORCA
0af03caefc is described below
commit 0af03caefc603c157ceec7e2ead115fa70cc8f5b
Author: zhoujiaqi <zh...@hashdata.cn>
AuthorDate: Tue Mar 4 16:51:55 2025 +0800
[ORCA] Support PAX AM in ORCA
ORCA can generate plans for PAX(Access method) because PAX is registered in `RetrieveRelStorageType`,
but PAX will be registered as `ErelstorageAppendOnlyCols`.
Use the `ao version` as an example, if we still keep the `get_ao_version` logic (currently deleted),
then PAX will not be able to generate `IndexOnlyScan`, cause it have have registered in
`pg_appendonly`(should not and won't).
The current commit adds `ErelstoragePAX` storage type, not only to identify the PAX in DXL/Expression,
but also to do some preparatory work for optimization of PAX. For example, PAX can use its own
statistics as AM callbacks which can help ORCA better calculate the costs of `SeqScan`.
---
src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp | 6 +++---
src/backend/gpopt/translate/CTranslatorRelcacheToDXL.cpp | 14 ++------------
src/backend/gporca/libgpdbcost/src/CCostModelGPDB.cpp | 16 ++++++++--------
.../libgpopt/include/gpopt/metadata/CTableDescriptor.h | 5 +++--
.../xforms/CXformDynamicIndexGet2DynamicIndexScan.cpp | 2 +-
.../libgpopt/src/xforms/CXformIndexGet2IndexScan.cpp | 2 +-
src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp | 4 ++--
.../libnaucrates/include/naucrates/dxl/xml/dxltokens.h | 1 +
.../libnaucrates/include/naucrates/md/IMDRelation.h | 6 ++++--
src/backend/gporca/libnaucrates/src/md/IMDRelation.cpp | 3 +++
.../libnaucrates/src/operators/CDXLOperatorFactory.cpp | 7 +++++++
src/backend/gporca/libnaucrates/src/xml/dxltokens.cpp | 2 ++
12 files changed, 37 insertions(+), 31 deletions(-)
diff --git a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp
index 76fde51238..b329655ac9 100644
--- a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp
+++ b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp
@@ -695,7 +695,7 @@ CTranslatorDXLToPlStmt::TranslateDXLTblScan(
plan->qual = security_query_quals;
}
- if (md_rel->IsAORowOrColTable())
+ if (md_rel->IsNonBlockTable())
{
CheckSafeTargetListForAOTables(plan->targetlist);
}
@@ -870,7 +870,7 @@ CTranslatorDXLToPlStmt::TranslateDXLIndexScan(
index_scan->indexorderdir = CTranslatorUtils::GetScanDirection(
physical_idx_scan_dxlop->GetIndexScanDir());
- if (md_rel->IsAORowOrColTable())
+ if (md_rel->IsNonBlockTable())
{
CheckSafeTargetListForAOTables(plan->targetlist);
}
@@ -4722,7 +4722,7 @@ CTranslatorDXLToPlStmt::TranslateDXLDml(
IMDId *mdid_target_table = phy_dml_dxlop->GetDXLTableDescr()->MDId();
const IMDRelation *md_rel = m_md_accessor->RetrieveRel(mdid_target_table);
- if (md_rel->IsAORowOrColTable())
+ if (md_rel->IsNonBlockTable())
{
isSplit = true; // AO tables are always use split updates
}
diff --git a/src/backend/gpopt/translate/CTranslatorRelcacheToDXL.cpp b/src/backend/gpopt/translate/CTranslatorRelcacheToDXL.cpp
index 09731ccc3b..4e1d1e8718 100644
--- a/src/backend/gpopt/translate/CTranslatorRelcacheToDXL.cpp
+++ b/src/backend/gpopt/translate/CTranslatorRelcacheToDXL.cpp
@@ -2545,19 +2545,9 @@ CTranslatorRelcacheToDXL::RetrieveRelStorageType(Relation rel)
case HEAP_TABLE_AM_OID:
rel_storage_type = IMDRelation::ErelstorageHeap;
break;
- // FIXME: need to add support for custom table am!!!
- //
- // Why 7047 here?
- // Because we defined a custom table am using columnar storage,
- // the orca optimizer does not support am other than HEAP/AO/AOCS. At present,
- // there is no way to extend orca to support custom table am. So here we use
- // the am_id(7047) we assigned to the custom table am, and let the orca optimizer
- // treat this columnar storage format as AOCS to generate an execution plan
- //
- // Why use the magic number 7047 instead of the macro definition?
- // Just to make it look like it doesn't make sense,
- // so others will notice that the logic needs to be refactored
case PAX_AM_OID:
+ rel_storage_type = IMDRelation::ErelstoragePAX;
+ break;
case AO_COLUMN_TABLE_AM_OID:
rel_storage_type = IMDRelation::ErelstorageAppendOnlyCols;
break;
diff --git a/src/backend/gporca/libgpdbcost/src/CCostModelGPDB.cpp b/src/backend/gporca/libgpdbcost/src/CCostModelGPDB.cpp
index aa940a7578..2bc469084b 100644
--- a/src/backend/gporca/libgpdbcost/src/CCostModelGPDB.cpp
+++ b/src/backend/gporca/libgpdbcost/src/CCostModelGPDB.cpp
@@ -1888,9 +1888,9 @@ CCostModelGPDB::CostIndexOnlyScan(CMemoryPool *mp GPOS_UNUSED, // mp
const CDouble dTableWidth =
CPhysicalScan::PopConvert(pop)->PstatsBaseTable()->Width();
- BOOL isAO = CPhysicalScan::PopConvert(exprhdl.Pop())
+ BOOL isNonBlock = CPhysicalScan::PopConvert(exprhdl.Pop())
->Ptabdesc()
- ->IsAORowOrColTable();
+ ->IsNonBlockTable();
CDouble dIndexFilterCostUnit =
pcmgpdb->GetCostModelParams()
@@ -1912,9 +1912,9 @@ CCostModelGPDB::CostIndexOnlyScan(CMemoryPool *mp GPOS_UNUSED, // mp
GPOS_ASSERT(0 < dIndexScanTupCostUnit);
GPOS_ASSERT(0 < dIndexScanTupRandomFactor);
- if (isAO)
+ if (isNonBlock)
{
- // AO specific costs related to index-scan/index-only-scan:
+ // AO(or PAX) specific costs related to index-scan/index-only-scan:
//
// * AO tables have a variable block size layout on disk (e.g. batch
// insert creates larger block than single row insert). However,
@@ -2005,7 +2005,7 @@ CCostModelGPDB::CostIndexOnlyScan(CMemoryPool *mp GPOS_UNUSED, // mp
CDouble dPartialVisFrac(1);
CDouble dCostVisibilityMapLookup(0);
- if (isAO)
+ if (isNonBlock)
{
dPartialVisFrac = 0;
}
@@ -2185,9 +2185,9 @@ CCostModelGPDB::CostBitmapTableScan(CMemoryPool *mp, CExpressionHandle &exprhdl,
pcmgpdb->GetCostModelParams()
->PcpLookup(CCostModelParamsGPDB::EcpBitmapPageCost)
->Get();
- BOOL isAOTable = CPhysicalScan::PopConvert(exprhdl.Pop())
+ BOOL isNonBlockTable = CPhysicalScan::PopConvert(exprhdl.Pop())
->Ptabdesc()
- ->IsAORowOrColTable();
+ ->IsNonBlockTable();
// some cost constants determined with the cal_bitmap_test.py script
CDouble c1_cost_per_row(0.03);
@@ -2214,7 +2214,7 @@ CCostModelGPDB::CostBitmapTableScan(CMemoryPool *mp, CExpressionHandle &exprhdl,
CDouble bitmapUnionCost = 0;
- if (!isAOTable && indexType == IMDIndex::EmdindBitmap && dNDV > 1.0)
+ if (!isNonBlockTable && indexType == IMDIndex::EmdindBitmap && dNDV > 1.0)
{
CDouble baseTableRows = CPhysicalScan::PopConvert(exprhdl.Pop())
->PstatsBaseTable()
diff --git a/src/backend/gporca/libgpopt/include/gpopt/metadata/CTableDescriptor.h b/src/backend/gporca/libgpopt/include/gpopt/metadata/CTableDescriptor.h
index 9ef2184600..29db4f2e40 100644
--- a/src/backend/gporca/libgpopt/include/gpopt/metadata/CTableDescriptor.h
+++ b/src/backend/gporca/libgpopt/include/gpopt/metadata/CTableDescriptor.h
@@ -239,10 +239,11 @@ public:
ULONG IndexCount();
BOOL
- IsAORowOrColTable() const
+ IsNonBlockTable() const
{
return m_erelstoragetype == IMDRelation::ErelstorageAppendOnlyCols ||
- m_erelstoragetype == IMDRelation::ErelstorageAppendOnlyRows;
+ m_erelstoragetype == IMDRelation::ErelstorageAppendOnlyRows ||
+ m_erelstoragetype == IMDRelation::ErelstoragePAX;
}
ULONG
diff --git a/src/backend/gporca/libgpopt/src/xforms/CXformDynamicIndexGet2DynamicIndexScan.cpp b/src/backend/gporca/libgpopt/src/xforms/CXformDynamicIndexGet2DynamicIndexScan.cpp
index 70678b9f26..6cf0e54aa9 100644
--- a/src/backend/gporca/libgpopt/src/xforms/CXformDynamicIndexGet2DynamicIndexScan.cpp
+++ b/src/backend/gporca/libgpopt/src/xforms/CXformDynamicIndexGet2DynamicIndexScan.cpp
@@ -50,7 +50,7 @@ CXformDynamicIndexGet2DynamicIndexScan::Exfp(CExpressionHandle &exprhdl) const
CLogicalDynamicIndexGet::PopConvert(exprhdl.Pop());
CTableDescriptor *ptabdesc = popGet->Ptabdesc();
- BOOL possible_ao_table = ptabdesc->IsAORowOrColTable() ||
+ BOOL possible_ao_table = ptabdesc->IsNonBlockTable() ||
ptabdesc->RetrieveRelStorageType() ==
IMDRelation::ErelstorageMixedPartitioned;
if (possible_ao_table || exprhdl.DeriveHasSubquery(0))
diff --git a/src/backend/gporca/libgpopt/src/xforms/CXformIndexGet2IndexScan.cpp b/src/backend/gporca/libgpopt/src/xforms/CXformIndexGet2IndexScan.cpp
index 6a02c08f49..256d8aa838 100644
--- a/src/backend/gporca/libgpopt/src/xforms/CXformIndexGet2IndexScan.cpp
+++ b/src/backend/gporca/libgpopt/src/xforms/CXformIndexGet2IndexScan.cpp
@@ -48,7 +48,7 @@ CXformIndexGet2IndexScan::Exfp(CExpressionHandle &exprhdl) const
CTableDescriptor *ptabdesc = popGet->Ptabdesc();
CIndexDescriptor *pindexdesc = popGet->Pindexdesc();
- BOOL possible_ao_table = ptabdesc->IsAORowOrColTable() ||
+ BOOL possible_ao_table = ptabdesc->IsNonBlockTable() ||
ptabdesc->RetrieveRelStorageType() ==
IMDRelation::ErelstorageMixedPartitioned;
diff --git a/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp b/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp
index 1cdfb4cae7..b1b12e45ce 100644
--- a/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp
+++ b/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp
@@ -1925,7 +1925,7 @@ CXformUtils::FIndexApplicable(CMemoryPool *mp, const IMDIndex *pmdindex,
IMDIndex::EmdindexType emdindtype,
IMDIndex::EmdindexType altindtype)
{
- BOOL possible_ao_table = pmdrel->IsAORowOrColTable() ||
+ BOOL possible_ao_table = pmdrel->IsNonBlockTable() ||
pmdrel->RetrieveRelStorageType() ==
IMDRelation::ErelstorageMixedPartitioned;
// GiST and Hash can match with either Btree or Bitmap indexes
@@ -3010,7 +3010,7 @@ CXformUtils::PexprBitmapSelectBestIndex(
pexprIndex->Release();
BOOL possible_ao_table =
- pmdrel->IsAORowOrColTable() ||
+ pmdrel->IsNonBlockTable() ||
pmdrel->RetrieveRelStorageType() ==
IMDRelation::ErelstorageMixedPartitioned;
diff --git a/src/backend/gporca/libnaucrates/include/naucrates/dxl/xml/dxltokens.h b/src/backend/gporca/libnaucrates/include/naucrates/dxl/xml/dxltokens.h
index 60c7d26940..5000a192ff 100644
--- a/src/backend/gporca/libnaucrates/include/naucrates/dxl/xml/dxltokens.h
+++ b/src/backend/gporca/libnaucrates/include/naucrates/dxl/xml/dxltokens.h
@@ -500,6 +500,7 @@ enum Edxltoken
EdxltokenRelStorageHeap,
EdxltokenRelStorageAppendOnlyCols,
EdxltokenRelStorageAppendOnlyRows,
+ EdxltokenRelStoragePAX,
EdxltokenRelStorageMixedPartitioned,
EdxltokenRelStorageForeign,
EdxltokenRelStorageCompositeType,
diff --git a/src/backend/gporca/libnaucrates/include/naucrates/md/IMDRelation.h b/src/backend/gporca/libnaucrates/include/naucrates/md/IMDRelation.h
index 819e85133f..036af05597 100644
--- a/src/backend/gporca/libnaucrates/include/naucrates/md/IMDRelation.h
+++ b/src/backend/gporca/libnaucrates/include/naucrates/md/IMDRelation.h
@@ -50,6 +50,7 @@ public:
ErelstorageHeap,
ErelstorageAppendOnlyCols,
ErelstorageAppendOnlyRows,
+ ErelstoragePAX,
ErelstorageForeign,
ErelstorageMixedPartitioned,
ErelstorageCompositeType,
@@ -189,11 +190,12 @@ public:
IMDRelation::Erelstoragetype rel_storage_type);
BOOL
- IsAORowOrColTable() const
+ IsNonBlockTable() const
{
Erelstoragetype st = RetrieveRelStorageType();
return st == ErelstorageAppendOnlyCols ||
- st == ErelstorageAppendOnlyRows;
+ st == ErelstorageAppendOnlyRows ||
+ st == ErelstoragePAX;
}
// get oid of foreign server for foreign table
diff --git a/src/backend/gporca/libnaucrates/src/md/IMDRelation.cpp b/src/backend/gporca/libnaucrates/src/md/IMDRelation.cpp
index 142fff98cf..badc102460 100644
--- a/src/backend/gporca/libnaucrates/src/md/IMDRelation.cpp
+++ b/src/backend/gporca/libnaucrates/src/md/IMDRelation.cpp
@@ -67,6 +67,9 @@ IMDRelation::GetStorageTypeStr(IMDRelation::Erelstoragetype rel_storage_type)
case ErelstorageAppendOnlyRows:
return CDXLTokens::GetDXLTokenStr(
EdxltokenRelStorageAppendOnlyRows);
+ case ErelstoragePAX:
+ return CDXLTokens::GetDXLTokenStr(
+ EdxltokenRelStoragePAX);
case ErelstorageForeign:
return CDXLTokens::GetDXLTokenStr(EdxltokenRelStorageForeign);
case ErelstorageMixedPartitioned:
diff --git a/src/backend/gporca/libnaucrates/src/operators/CDXLOperatorFactory.cpp b/src/backend/gporca/libnaucrates/src/operators/CDXLOperatorFactory.cpp
index 30b9a35485..d309cb63f2 100644
--- a/src/backend/gporca/libnaucrates/src/operators/CDXLOperatorFactory.cpp
+++ b/src/backend/gporca/libnaucrates/src/operators/CDXLOperatorFactory.cpp
@@ -3783,6 +3783,13 @@ CDXLOperatorFactory::ParseRelationStorageType(const XMLCh *xml_val)
return IMDRelation::ErelstorageAppendOnlyRows;
}
+ if (0 == XMLString::compareString(
+ xml_val,
+ CDXLTokens::XmlstrToken(EdxltokenRelStoragePAX)))
+ {
+ return IMDRelation::ErelstoragePAX;
+ }
+
if (0 == XMLString::compareString(
xml_val, CDXLTokens::XmlstrToken(EdxltokenRelStorageForeign)))
{
diff --git a/src/backend/gporca/libnaucrates/src/xml/dxltokens.cpp b/src/backend/gporca/libnaucrates/src/xml/dxltokens.cpp
index 90590092bf..01055bab44 100644
--- a/src/backend/gporca/libnaucrates/src/xml/dxltokens.cpp
+++ b/src/backend/gporca/libnaucrates/src/xml/dxltokens.cpp
@@ -550,6 +550,8 @@ CDXLTokens::Init(CMemoryPool *mp)
GPOS_WSZ_LIT("AppendOnly, Column-oriented")},
{EdxltokenRelStorageAppendOnlyRows,
GPOS_WSZ_LIT("AppendOnly, Row-oriented")},
+ {EdxltokenRelStoragePAX,
+ GPOS_WSZ_LIT("PAX")},
{EdxltokenRelStorageMixedPartitioned, GPOS_WSZ_LIT("MixedPartitioned")},
{EdxltokenRelStorageForeign, GPOS_WSZ_LIT("Foreign")},
{EdxltokenRelStorageCompositeType, GPOS_WSZ_LIT("Composite")},
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cloudberry.apache.org
For additional commands, e-mail: commits-help@cloudberry.apache.org