You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudberry.apache.org by dj...@apache.org on 2025/12/18 08:21:16 UTC
(cloudberry-pxf) 11/37: updated ProjectionInfo related code for GP7 and code refactor
This is an automated email from the ASF dual-hosted git repository.
djwang pushed a commit to branch merge-with-upstream
in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
commit 6fa21e7cd6196d13eb5ee8250b4e13c01f575454
Author: Himanshu Pandey <pa...@vmware.com>
AuthorDate: Thu Sep 22 10:42:40 2022 -0700
updated ProjectionInfo related code for GP7 and code refactor
---
external-table/src/pxfheaders.c | 245 ++++++++++++++++++++++++++++++----------
1 file changed, 185 insertions(+), 60 deletions(-)
diff --git a/external-table/src/pxfheaders.c b/external-table/src/pxfheaders.c
index d07a9595..a9bd3661 100644
--- a/external-table/src/pxfheaders.c
+++ b/external-table/src/pxfheaders.c
@@ -23,6 +23,7 @@
#if PG_VERSION_NUM >= 120000
#include "access/external.h"
#include "extension/gp_exttable_fdw/extaccess.h"
+#include "executor/execExpr.h"
#else
#include "access/fileam.h"
#include "catalog/pg_exttable.h"
@@ -39,7 +40,12 @@ static char *get_format_name(ExtTableEntry *exttbl);
static char *getFormatterString(ExtTableEntry *exttbl);
static bool isFormatterPxfDelimited(ExtTableEntry *exttbl);
static bool isFormatterPxfWritable(ExtTableEntry *exttbl);
-static void add_projection_desc_httpheaders(CHURL_HEADERS headers, ProjectionInfo *projInfo, List *qualsAttributes, Relation rel);
+#if PG_VERSION_NUM >= 120000
+static void add_projection_desc_httpheader_pg12(CHURL_HEADERS headers, ProjectionInfo *projInfo, List *qualsAttributes, Relation rel);
+#else
+static void add_projection_desc_httpheader_pg94(CHURL_HEADERS headers, ProjectionInfo *projInfo, List *qualsAttributes, Relation rel);
+#endif
+
static bool add_attnums_from_targetList(Node *node, List *attnums);
static void add_projection_index_header(CHURL_HEADERS pVoid, StringInfoData data, int attno, char number[32]);
static List *appendCopyEncodingOptionToList(List *copyFmtOpts, int encoding);
@@ -155,7 +161,11 @@ build_http_headers(PxfInputData *input)
if (qualsAreSupported &&
(qualsAttributes != NIL || list_length(input->quals) == 0))
{
- add_projection_desc_httpheaders(headers, proj_info, qualsAttributes, rel);
+ #if PG_VERSION_NUM >= 120000
+ add_projection_desc_httpheader_pg12(headers, proj_info, qualsAttributes, rel);
+ #else
+ add_projection_desc_httpheader_pg94(headers, proj_info, qualsAttributes, rel);
+ #endif
}
else
{
@@ -272,7 +282,7 @@ add_tuple_desc_httpheader(CHURL_HEADERS headers, Relation rel)
for (i = 0, attrIx = 0; i < tuple->natts; ++i)
{
#if PG_VERSION_NUM >= 120000
- Form_pg_attribute attribute = &tuple->attrs[i];
+ FormData_pg_attribute *attribute = &tuple->attrs[i];
#else
FormData_pg_attribute *attribute = tuple->attrs[i];
#endif
@@ -494,8 +504,10 @@ int getNumSimpleVars(ProjectionInfo *projInfo)
* indices will be 0, 2. This is because we use 0-based indexing and because
* col2 was dropped, the indices for col3 and col4 get shifted by -1.
*/
+
+#if PG_VERSION_NUM < 120000
static void
-add_projection_desc_httpheaders(CHURL_HEADERS headers,
+add_projection_desc_httpheader_pg94(CHURL_HEADERS headers,
ProjectionInfo *projInfo,
List *qualsAttributes,
Relation rel)
@@ -504,22 +516,11 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers,
int dropped_count;
int number;
int numTargetList;
-#if PG_VERSION_NUM < 90400 || PG_VERSION_NUM == 120000
- int numSimpleVars = 0;
+#if PG_VERSION_NUM < 90400
+ int numSimpleVars;
#endif
char long_number[sizeof(int32) * 8];
- /* FIXME: to get it to compile assign it to NULL */
- // pi_varNumbers is not available anymore in the postgters code
- // https://doxygen.postgresql.org/structProjectionInfo.html
- //int *varNumbers = projInfo->pi_varNumbers;
-#if PG_VERSION_NUM < 120000 // This will work for 6.x and lower versions of GPDB
- int *varNumbers = projInfo->pi_varNumbers;
- List *targetList = projInfo->pi_targetlist;
-#else
- int *varNumbers = NULL; //TODO Need to find a proper value here
- List *targetList = (List *)projInfo->pi_state.resultslot;
-#endif
-
+ int *varNumbers = projInfo->pi_varNumbers;
Bitmapset *attrs_used;
StringInfoData formatter;
List *targetList; // targetList from the query plan
@@ -528,18 +529,14 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers,
// STEP 0: initialize variables
initStringInfo(&formatter);
numTargetList = 0;
+
#if PG_VERSION_NUM >= 90400
/*
* Non-simpleVars are added to the targetlist
* we use expression_tree_walker to access attrno information
* we do it through a helper function add_attnums_from_targetList
*/
- /* FIXME: commenting this out for compilation success */
- // pi_targetlist is not available anymore in the postgters code
- // https://doxygen.postgresql.org/structProjectionInfo.html
- //if (projInfo->pi_targetlist)
-
- if (targetList)
+ if (projInfo->pi_targetlist)
{
#else
numSimpleVars = 0;
@@ -553,17 +550,10 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers,
List *l = lappend_int(NIL, 0);
ListCell *lc1;
- /* FIXME: commenting this out to make it compile */
-
- foreach(lc1, targetList)
+ foreach(lc1, projInfo->pi_targetlist)
{
- #if PG_VERSION_NUM >= 120000
- ExprState *gstate = (ExprState *) lfirst(lc1);
- add_attnums_from_targetList((Node *) gstate->expr, l);
- #else
GenericExprState *gstate = (GenericExprState *) lfirst(lc1);
add_attnums_from_targetList((Node *) gstate->arg->expr, l);
- #endif
}
foreach(lc1, l)
@@ -579,29 +569,19 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers,
list_free(l);
}
-#if PG_VERSION_NUM < 90400 || PG_VERSION_NUM >= 120000
+#if PG_VERSION_NUM < 90400
else
{
- #if PG_VERSION_NUM >= 120000
- numSimpleVars = list_length(targetList);
- #else
- numSimpleVars = list_length(projInfo->pi_targetlist);
- #endif
+ numSimpleVars = list_length(projInfo->pi_targetlist);
}
#endif
number = numTargetList +
-
-
- #if PG_VERSION_NUM >= 90400 && PG_VERSION_NUM < 120000
- // FIXME: Commenting this out for compilation success
- // pi_numSimpleVars is not available anymore in the postgters 12 code
- // https://doxygen.postgresql.org/structProjectionInfo.html
- projInfo->pi_numSimpleVars +
- #else
- numSimpleVars +
- #endif
-
+#if PG_VERSION_NUM >= 90400
+ projInfo->pi_numSimpleVars +
+#else
+ numSimpleVars +
+#endif
list_length(qualsAttributes);
if (number == 0)
return;
@@ -612,13 +592,11 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers,
pg_ltoa(number, long_number);
churl_headers_append(headers, "X-GP-ATTRS-PROJ", long_number);
-
-#if PG_VERSION_NUM >= 90400 && PG_VERSION_NUM < 120000
- /* FIXME: commenting out to get compile to work */
- //for (i = 0; i < projInfo->pi_numSimpleVars; i++)
- for (i = 0; i < projInfo->pi_numSimpleVars; i++)
+#if PG_VERSION_NUM >= 90400
+ for (i = 0; i < projInfo->pi_numSimpleVars; i++)
#else
for (i = 0; varNumbers && i < numSimpleVars; i++)
+#endif
{
attrs_used =
bms_add_member(attrs_used,
@@ -642,13 +620,8 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers,
for (i = 1; i <= tupdesc->natts; i++)
{
- #if PG_VERSION_NUM >= 120000
- Form_pg_attribute attr = &tupdesc->attrs[i-1];
- #else
- FormData_pg_attribute *attr = tupdesc->attrs[i-1];
- #endif
/* Ignore dropped attributes. */
- if (attr->attisdropped)
+ if (tupdesc->attrs[i - 1]->attisdropped)
{
/* keep a counter of the number of dropped attributes */
droppedCount++;
@@ -675,6 +648,158 @@ add_projection_desc_httpheaders(CHURL_HEADERS headers,
pfree(formatter.data);
bms_free(attrs_used);
}
+#endif
+
+/*
+ * Report projection description to the remote component, the indices of
+ * dropped columns do not get reported, as if they never existed, and
+ * column indices that follow dropped columns will be shifted by the number
+ * of dropped columns that precede it. For example,
+ *
+ * ---------------------------------------------
+ * | col1 | col2 (dropped) | col3 | col4 |
+ * ---------------------------------------------
+ *
+ * Let's assume that col1 and col4 are projected, the reported projected
+ * indices will be 0, 2. This is because we use 0-based indexing and because
+ * col2 was dropped, the indices for col3 and col4 get shifted by -1.
+ */
+
+#if PG_VERSION_NUM >= 120000
+static void
+add_projection_desc_httpheader_pg12(CHURL_HEADERS headers,
+ ProjectionInfo *projInfo,
+ List *qualsAttributes,
+ Relation rel)
+{
+ int i;
+ int dropped_count;
+ int number;
+ int numTargetList;
+ char long_number[sizeof(int32) * 8];
+ //int *varNumbers = projInfo->pi_varNumbers;
+ Bitmapset *attrs_used;
+ StringInfoData formatter;
+ TupleDesc tupdesc;
+ ExprEvalStep step;
+ initStringInfo(&formatter);
+ numTargetList = 0;
+
+ List *targetList = (List *) projInfo->pi_state.expr;
+ int numSimpleVars = 0;
+
+#if PG_VERSION_NUM >= 90400
+
+ int numNonSimpleVars = 0;
+
+ for (int i = 0; i < projInfo->pi_state.steps_len; i++)
+ {
+ step = projInfo->pi_state.steps[i];
+ if (step.opcode == EEOP_ASSIGN_INNER_VAR ||
+ step.opcode == EEOP_ASSIGN_OUTER_VAR ||
+ step.opcode == EEOP_ASSIGN_SCAN_VAR)
+ numSimpleVars++;
+ else if (step.opcode == EEOP_ASSIGN_TMP_MAKE_RO ||
+ step.opcode == EEOP_ASSIGN_TMP)
+ numNonSimpleVars++;
+ }
+ int * varNumbers = &numNonSimpleVars;
+ /*
+ * Non-simpleVars are added to the targetlist
+ * we use expression_tree_walker to access attrno information
+ * we do it through a helper function add_attnums_from_targetList
+ */
+ if (targetList)
+ {
+#else
+
+ if (numNonSimpleVars > 0)
+ {
+ /*
+ * When there are not just simple Vars we need to
+ * walk the tree to get attnums
+ */
+#endif
+ List *l = lappend_int(NIL, 0);
+ ListCell *lc1;
+
+ foreach(lc1, targetList)
+ {
+ ExprState *gstate = (ExprState *) lfirst(lc1);
+ add_attnums_from_targetList( (Node *) gstate, l);
+ }
+
+ foreach(lc1, l)
+ {
+ int attno = lfirst_int(lc1);
+ if (attno > InvalidAttrNumber)
+ {
+ add_projection_index_header(headers,
+ formatter, attno - 1, long_number);
+ numTargetList++;
+ }
+ }
+
+ list_free(l);
+ }
+
+ number = numTargetList + numSimpleVars + list_length(qualsAttributes);
+ if (number == 0)
+ return;
+
+ attrs_used = NULL;
+
+ /* Convert the number of projection columns to a string */
+ pg_ltoa(number, long_number);
+ churl_headers_append(headers, "X-GP-ATTRS-PROJ", long_number);
+
+ for (i = 0; varNumbers && i < numSimpleVars ; i++)
+ {
+ attrs_used =
+ bms_add_member(attrs_used,
+ varNumbers[i] - FirstLowInvalidHeapAttributeNumber);
+ }
+
+ ListCell *attribute = NULL;
+
+ /*
+ * AttrNumbers coming from quals
+ */
+ foreach(attribute, qualsAttributes)
+ {
+ AttrNumber attrNumber = (AttrNumber) lfirst_int(attribute);
+ attrs_used =
+ bms_add_member(attrs_used,
+ attrNumber + 1 - FirstLowInvalidHeapAttributeNumber);
+ }
+
+ tupdesc = RelationGetDescr(rel);
+ dropped_count = 0;
+
+ for (i = 1; i <= tupdesc->natts; i++)
+ {
+ /* Ignore dropped attributes. */
+ if (tupdesc->attrs[i - 1].attisdropped)
+ {
+ /* keep a counter of the number of dropped attributes */
+ dropped_count++;
+ continue;
+ }
+
+ if (bms_is_member(i - FirstLowInvalidHeapAttributeNumber, attrs_used))
+ {
+ /* Shift the column index by the running dropped_count */
+ add_projection_index_header(headers, formatter,
+ i - 1 - dropped_count, long_number);
+ }
+ }
+
+ list_free(qualsAttributes);
+ pfree(formatter.data);
+ bms_free(attrs_used);
+}
+#endif
+
/*
* Adds the projection index header for the given attno
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cloudberry.apache.org
For additional commands, e-mail: commits-help@cloudberry.apache.org