You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by hu...@apache.org on 2017/12/01 08:52:40 UTC
incubator-hawq git commit: HAWQ-1555. Add access interfaces for
protocol and format in pluggable storage framework
Repository: incubator-hawq
Updated Branches:
refs/heads/master 5e152951d -> 96c13f7ba
HAWQ-1555. Add access interfaces for protocol and format in pluggable storage framework
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/96c13f7b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/96c13f7b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/96c13f7b
Branch: refs/heads/master
Commit: 96c13f7bacf46e088114dfbf38ad8006e0dfc92f
Parents: 5e15295
Author: Chiyang Wan <ch...@gmail.com>
Authored: Tue Nov 28 09:16:48 2017 +0800
Committer: Ruilong Huo <hu...@163.com>
Committed: Fri Dec 1 16:52:13 2017 +0800
----------------------------------------------------------------------
src/backend/access/external/Makefile | 3 +-
src/backend/access/external/fileam.c | 37 ++
src/backend/access/external/plugstorage.c | 553 +++++++++++++++++++++++++
src/include/access/fileam.h | 2 +
src/include/access/plugstorage.h | 222 ++++++++++
src/include/access/plugstorage_utils.h | 59 +++
src/include/nodes/nodes.h | 3 +
7 files changed, 878 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/96c13f7b/src/backend/access/external/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/access/external/Makefile b/src/backend/access/external/Makefile
index bc043dc..cc52f2f 100644
--- a/src/backend/access/external/Makefile
+++ b/src/backend/access/external/Makefile
@@ -27,7 +27,8 @@ top_builddir = ../../../..
include $(top_builddir)/src/Makefile.global
OBJS = fileam.o url.o libchurl.o hd_work_mgr.o pxfuriparser.o pxfheaders.o \
-pxfmasterapi.o ha_config.o pxfcomutils.o pxfutils.o pxffilters.o pxfanalyze.o
+pxfmasterapi.o ha_config.o pxfcomutils.o pxfutils.o pxffilters.o pxfanalyze.o \
+plugstorage.o
include $(top_srcdir)/src/backend/common.mk
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/96c13f7b/src/backend/access/external/fileam.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/fileam.c b/src/backend/access/external/fileam.c
index f77b29e..099dae5 100644
--- a/src/backend/access/external/fileam.c
+++ b/src/backend/access/external/fileam.c
@@ -2310,6 +2310,43 @@ strtokx2(const char *s,
return start;
}
+char *getExtTblFormatterTypeInFmtOptsStr(char *fmtStr)
+{
+ const char *whitespace = " \t\n\r";
+ const char *quote = "'";
+ int encoding = GetDatabaseEncoding();
+
+ char *key = strtokx2(fmtStr, whitespace, NULL, NULL,
+ 0, false, true, encoding);
+ char *val = strtokx2(NULL, whitespace, NULL, quote,
+ 0, false, true, encoding);
+
+ while (key && val)
+ {
+ if (pg_strncasecmp(key, "formatter", strlen("formatter")) == 0)
+ {
+ return pstrdup(val);
+ }
+
+ key = strtokx2(NULL, whitespace, NULL, NULL,
+ 0, false, false, encoding);
+ val = strtokx2(NULL, whitespace, NULL, quote,
+ 0, false, true, encoding);
+ }
+
+ return NULL;
+}
+
+char *getExtTblFormatterTypeInFmtOptsList(List *fmtOpts)
+{
+ /* formatter always is at the begin the fmtOpts */
+ char *formatterStr = pstrdup((char *) strVal(linitial(fmtOpts)));
+ char *formatterName = getExtTblFormatterTypeInFmtOptsStr(formatterStr);
+ pfree(formatterStr);
+
+ return formatterName;
+}
+
/*
* parseFormatString
*
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/96c13f7b/src/backend/access/external/plugstorage.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/plugstorage.c b/src/backend/access/external/plugstorage.c
new file mode 100644
index 0000000..21dd51e
--- /dev/null
+++ b/src/backend/access/external/plugstorage.c
@@ -0,0 +1,553 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*-------------------------------------------------------------------------
+ *
+ * plugstorage.c
+ *
+ * Pluggable storage implementation. Support external table for now.
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "access/plugstorage.h"
+#include "catalog/pg_type.h"
+#include "catalog/pg_proc.h"
+#include "catalog/pg_exttable.h"
+#include "nodes/value.h"
+#include "parser/parse_func.h"
+
+/*
+ * getExternalTableTypeIn(List/Str)
+ *
+ * Return the table type for a given external table
+ *
+ * Currently it is an implementation with performance cost due to the
+ * fact that the format types for TEXT, CSV, and ORC external tables
+ * with customer protocol (HDFS) are all 'b' in pg_exttable catalog
+ * table. It needs to visit the format options to get the table type.
+ */
+void getExternalTableTypeInList(const char formatType,
+ List *formatOptions,
+ int *formatterType,
+ char **formatterName)
+{
+ if (fmttype_is_text(formatType))
+ {
+ *formatterType = ExternalTableType_TEXT;
+ *formatterName = NULL;
+ }
+ else if (fmttype_is_csv(formatType))
+ {
+ *formatterType = ExternalTableType_CSV;
+ *formatterName = NULL;
+ }
+ else if (fmttype_is_custom(formatType))
+ {
+ Assert(formatOptions);
+
+ *formatterName = getExtTblFormatterTypeInFmtOptsList(formatOptions);
+
+ *formatterType = ExternalTableType_PLUG;
+ }
+ else
+ {
+ *formatterType = ExternalTableType_Invalid;
+ *formatterName = NULL;
+ elog(ERROR, "undefined external table format type: %c", formatType);
+ }
+}
+
+void getExternalTableTypeInStr(const char formatType,
+ char *formatOptions,
+ int *formatterType,
+ char **formatterName)
+{
+ if (fmttype_is_text(formatType))
+ {
+ *formatterType = ExternalTableType_TEXT;
+ *formatterName = NULL;
+ }
+ else if (fmttype_is_csv(formatType))
+ {
+ *formatterType = ExternalTableType_CSV;
+ *formatterName = NULL;
+ }
+ else if (fmttype_is_custom(formatType))
+ {
+ Assert(formatOptions);
+
+ *formatterName = getExtTblFormatterTypeInFmtOptsStr(formatOptions);
+ Assert(*formatterName);
+
+ *formatterType = ExternalTableType_PLUG;
+ }
+ else
+ {
+ *formatterType = ExternalTableType_Invalid;
+ *formatterName = NULL;
+ elog(ERROR, "undefined external table format type: %c", formatType);
+ }
+}
+
+/*
+ * Check if values for options of custom external table are valid
+ */
+void checkPlugStorageFormatOption(char **opt,
+ const char *key,
+ const char *val,
+ const bool needopt,
+ const int nvalidvals,
+ const char **validvals)
+{
+ Assert(opt);
+
+ /* check if need to check option */
+ if (!needopt)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("redundant option %s", key),
+ errOmitLocation(true)));
+ }
+
+ /* check if option is redundant */
+ if (*opt)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("conflicting or redundant options \"%s\"", key),
+ errOmitLocation(true)));
+ }
+
+ *opt = val;
+
+ /* check if value for option is valid */
+ bool valid = false;
+ for (int i = 0; i < nvalidvals; i++)
+ {
+ if (strncasecmp(*opt, validvals[i], strlen(validvals[i])) == 0)
+ {
+ valid = true;
+ }
+ }
+
+ if (!valid && nvalidvals > 0)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("invalid value for option %s: \"%s\"", key, val),
+ errOmitLocation(true)));
+ }
+}
+
+Oid LookupPlugStorageValidatorFunc(char *formatter,
+ char *validator)
+{
+ List* funcname = NIL;
+ Oid procOid = InvalidOid;
+ Oid argList[1];
+ Oid returnOid;
+
+ elog(DEBUG3, "find validator function for %s_%s", formatter, validator);
+
+ char *new_func_name = (char *)palloc0(strlen(formatter)+strlen(validator) + 2);
+ sprintf(new_func_name, "%s_%s", formatter, validator);
+ funcname = lappend(funcname, makeString(new_func_name));
+ returnOid = VOIDOID;
+ procOid = LookupFuncName(funcname, 0, argList, true);
+
+ pfree(new_func_name);
+
+ return procOid;
+}
+
+void InvokePlugStorageValidationFormatInterfaces(Oid procOid,
+ char *formatName)
+{
+
+ PlugStorageValidatorData psvdata;
+ FmgrInfo psvfunc;
+ FunctionCallInfoData fcinfo;
+
+ fmgr_info(procOid, &psvfunc);
+
+ psvdata.type = T_PlugStorageValidatorData;
+ psvdata.format_name = formatName;
+
+ InitFunctionCallInfoData(fcinfo,
+ &psvfunc,
+ 0,
+ (Node *)(&psvdata),
+ NULL);
+
+ FunctionCallInvoke(&fcinfo);
+
+ if (fcinfo.isnull)
+ {
+ elog(ERROR, "validator function %u returned NULL",
+ fcinfo.flinfo->fn_oid);
+ }
+}
+
+void InvokePlugStorageValidationFormatOptions(Oid procOid,
+ List *formatOptions,
+ char *formatStr,
+ bool isWritable)
+{
+
+ PlugStorageValidatorData psvdata;
+ FmgrInfo psvfunc;
+ FunctionCallInfoData fcinfo;
+
+ fmgr_info(procOid, &psvfunc);
+
+ psvdata.type = T_PlugStorageValidatorData;
+ psvdata.format_opts = formatOptions;
+ psvdata.format_str = formatStr;
+ psvdata.is_writable = isWritable;
+
+ InitFunctionCallInfoData(fcinfo,
+ &psvfunc,
+ 0,
+ (Node *)(&psvdata),
+ NULL);
+
+ FunctionCallInvoke(&fcinfo);
+
+ if (fcinfo.isnull)
+ {
+ elog(ERROR, "validator function %u returned NULL",
+ fcinfo.flinfo->fn_oid);
+ }
+}
+
+void InvokePlugStorageValidationFormatEncodings(Oid procOid,
+ char *encodingName)
+{
+
+ PlugStorageValidatorData psvdata;
+ FmgrInfo psvfunc;
+ FunctionCallInfoData fcinfo;
+
+ fmgr_info(procOid, &psvfunc);
+
+ psvdata.type = T_PlugStorageValidatorData;
+ psvdata.encoding_name = encodingName;
+
+ InitFunctionCallInfoData(fcinfo,
+ &psvfunc,
+ 0,
+ (Node *)(&psvdata),
+ NULL);
+
+ FunctionCallInvoke(&fcinfo);
+
+ if (fcinfo.isnull)
+ {
+ elog(ERROR, "validator function %u returned NULL",
+ fcinfo.flinfo->fn_oid);
+ }
+}
+
+void InvokePlugStorageValidationFormatDataTypes(Oid procOid,
+ TupleDesc tupDesc)
+{
+ PlugStorageValidatorData psvdata;
+ FmgrInfo psvfunc;
+ FunctionCallInfoData fcinfo;
+
+ fmgr_info(procOid, &psvfunc);
+
+ psvdata.type = T_PlugStorageValidatorData;
+ psvdata.tuple_desc = tupDesc;
+
+ InitFunctionCallInfoData(fcinfo,
+ &psvfunc,
+ 0,
+ (Node *)(&psvdata),
+ NULL);
+
+ FunctionCallInvoke(&fcinfo);
+
+ if (fcinfo.isnull)
+ {
+ elog(ERROR, "validator function %u returned NULL",
+ fcinfo.flinfo->fn_oid);
+ }
+}
+
+FileScanDesc InvokePlugStorageFormatBeginScan(FmgrInfo *func,
+ ExternalScan *extScan,
+ ScanState *scanState,
+ Relation relation,
+ int formatterType,
+ char *formatterName)
+{
+ PlugStorageData psdata;
+ FunctionCallInfoData fcinfo;
+
+ psdata.type = T_PlugStorageData;
+ psdata.ps_ext_scan = extScan;
+ psdata.ps_scan_state = scanState;
+ psdata.ps_relation = relation;
+ psdata.ps_formatter_type = formatterType;
+ psdata.ps_formatter_name = formatterName;
+
+ InitFunctionCallInfoData(fcinfo,
+ func,
+ 0,
+ (Node *)(&psdata),
+ NULL);
+
+ FunctionCallInvoke(&fcinfo);
+
+ if (fcinfo.isnull)
+ {
+ elog(ERROR, "function %u returned NULL",
+ fcinfo.flinfo->fn_oid);
+ }
+
+ FileScanDesc fileScanDesc = psdata.ps_file_scan_desc;
+
+ return fileScanDesc;
+}
+
+ExternalSelectDesc InvokePlugStorageFormatGetNextInit(FmgrInfo *func,
+ PlanState *planState,
+ ExternalScanState *extScanState)
+{
+ PlugStorageData psdata;
+ FunctionCallInfoData fcinfo;
+
+ psdata.type = T_PlugStorageData;
+ psdata.ps_plan_state = planState;
+ psdata.ps_ext_scan_state = extScanState;
+
+ InitFunctionCallInfoData(fcinfo,
+ func,
+ 0,
+ (Node *)(&psdata),
+ NULL);
+
+ FunctionCallInvoke(&fcinfo);
+
+ if (fcinfo.isnull)
+ {
+ elog(ERROR, "function %u returned NULL",
+ fcinfo.flinfo->fn_oid);
+ }
+
+ ExternalSelectDesc extSelectDesc = psdata.ps_ext_select_desc;
+
+ return extSelectDesc;
+}
+
+bool InvokePlugStorageFormatGetNext(FmgrInfo *func,
+ FileScanDesc fileScanDesc,
+ ScanDirection scanDirection,
+ ExternalSelectDesc extSelectDesc,
+ ScanState *scanState,
+ TupleTableSlot *tupTableSlot)
+{
+ PlugStorageData psdata;
+ FunctionCallInfoData fcinfo;
+
+ psdata.type = T_PlugStorageData;
+ psdata.ps_file_scan_desc = fileScanDesc;
+ psdata.ps_scan_direction = scanDirection;
+ psdata.ps_ext_select_desc = extSelectDesc;
+ psdata.ps_scan_state = scanState;
+ psdata.ps_tuple_table_slot = tupTableSlot;
+
+ InitFunctionCallInfoData(fcinfo,
+ func,
+ 0,
+ (Node *)(&psdata),
+ NULL);
+
+ FunctionCallInvoke(&fcinfo);
+
+ if (fcinfo.isnull)
+ {
+ elog(ERROR, "function %u returned NULL",
+ fcinfo.flinfo->fn_oid);
+ }
+
+ bool has_tuple = psdata.ps_has_tuple;
+
+ return has_tuple;
+}
+
+void InvokePlugStorageFormatReScan(FmgrInfo *func,
+ FileScanDesc fileScanDesc)
+{
+ PlugStorageData psdata;
+ FunctionCallInfoData fcinfo;
+
+ psdata.type = T_PlugStorageData;
+ psdata.ps_file_scan_desc = fileScanDesc;
+
+ InitFunctionCallInfoData(fcinfo,
+ func,
+ 0,
+ (Node *)(&psdata),
+ NULL);
+
+ FunctionCallInvoke(&fcinfo);
+
+ if (fcinfo.isnull)
+ {
+ elog(ERROR, "function %u returned NULL",
+ fcinfo.flinfo->fn_oid);
+ }
+}
+
+void InvokePlugStorageFormatEndScan(FmgrInfo *func,
+ FileScanDesc fileScanDesc)
+{
+ PlugStorageData psdata;
+ FunctionCallInfoData fcinfo;
+
+ psdata.type = T_PlugStorageData;
+ psdata.ps_file_scan_desc = fileScanDesc;
+
+ InitFunctionCallInfoData(fcinfo,
+ func,
+ 0,
+ (Node *)(&psdata),
+ NULL);
+
+ FunctionCallInvoke(&fcinfo);
+
+ if (fcinfo.isnull)
+ {
+ elog(ERROR, "function %u returned NULL",
+ fcinfo.flinfo->fn_oid);
+ }
+}
+
+void InvokePlugStorageFormatStopScan(FmgrInfo *func,
+ FileScanDesc fileScanDesc)
+{
+ PlugStorageData psdata;
+ FunctionCallInfoData fcinfo;
+
+ psdata.type = T_PlugStorageData;
+ psdata.ps_file_scan_desc = fileScanDesc;
+
+ InitFunctionCallInfoData(fcinfo,
+ func,
+ 0,
+ (Node *)(&psdata),
+ NULL);
+
+ FunctionCallInvoke(&fcinfo);
+
+ if (fcinfo.isnull)
+ {
+ elog(ERROR, "function %u returned NULL",
+ fcinfo.flinfo->fn_oid);
+ }
+}
+
+ExternalInsertDesc InvokePlugStorageFormatInsertInit(FmgrInfo *func,
+ Relation relation,
+ int formatterType,
+ char *formatterName)
+{
+ PlugStorageData psdata;
+ FunctionCallInfoData fcinfo;
+
+ psdata.type = T_PlugStorageData;
+ psdata.ps_relation = relation;
+ psdata.ps_formatter_type = formatterType;
+ psdata.ps_formatter_name = formatterName;
+
+ InitFunctionCallInfoData(fcinfo,
+ func,
+ 0,
+ (Node *)(&psdata),
+ NULL);
+
+ FunctionCallInvoke(&fcinfo);
+
+ if (fcinfo.isnull)
+ {
+ elog(ERROR, "function %u returned NULL",
+ fcinfo.flinfo->fn_oid);
+ }
+
+ ExternalInsertDesc extInsertDesc = psdata.ps_ext_insert_desc;
+
+ return extInsertDesc;
+}
+
+Oid InvokePlugStorageFormatInsert(FmgrInfo *func,
+ ExternalInsertDesc extInsertDesc,
+ TupleTableSlot *tupTableSlot)
+{
+ PlugStorageData psdata;
+ FunctionCallInfoData fcinfo;
+
+ psdata.type = T_PlugStorageData;
+ psdata.ps_ext_insert_desc = extInsertDesc;
+ psdata.ps_tuple_table_slot = tupTableSlot;
+
+ InitFunctionCallInfoData(fcinfo,
+ func,
+ 0,
+ (Node *)(&psdata),
+ NULL);
+
+ FunctionCallInvoke(&fcinfo);
+
+ if (fcinfo.isnull)
+ {
+ elog(ERROR, "function %u returned NULL",
+ fcinfo.flinfo->fn_oid);
+ }
+
+ Oid tuple_oid = psdata.ps_tuple_oid;
+
+ return tuple_oid;
+}
+
+void InvokePlugStorageFormatInsertFinish(FmgrInfo *func,
+ ExternalInsertDesc extInsertDesc)
+{
+ PlugStorageData psdata;
+ FunctionCallInfoData fcinfo;
+
+ psdata.type = T_PlugStorageData;
+ psdata.ps_ext_insert_desc = extInsertDesc;
+
+ InitFunctionCallInfoData(fcinfo,
+ func,
+ 0,
+ (Node *)(&psdata),
+ NULL);
+
+ FunctionCallInvoke(&fcinfo);
+
+ if (fcinfo.isnull)
+ {
+ elog(ERROR, "function %u returned NULL",
+ fcinfo.flinfo->fn_oid);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/96c13f7b/src/include/access/fileam.h
----------------------------------------------------------------------
diff --git a/src/include/access/fileam.h b/src/include/access/fileam.h
index 713c185..aee9979 100644
--- a/src/include/access/fileam.h
+++ b/src/include/access/fileam.h
@@ -100,5 +100,7 @@ extern void external_set_env_vars(extvar_t *extvar, char* uri, bool csv, char* e
extern void AtAbort_ExtTables(void);
char* linenumber_atoi(char buffer[20],int64 linenumber);
+extern char *getExtTblFormatterTypeInFmtOptsStr(char *fmtStr);
+extern char *getExtTblFormatterTypeInFmtOptsList(List *fmtOpts);
#endif /* FILEAM_H */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/96c13f7b/src/include/access/plugstorage.h
----------------------------------------------------------------------
diff --git a/src/include/access/plugstorage.h b/src/include/access/plugstorage.h
new file mode 100644
index 0000000..7450ba1
--- /dev/null
+++ b/src/include/access/plugstorage.h
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*-------------------------------------------------------------------------
+ *
+ * plugstorage.h
+ *
+ * Pluggable storage definitions. Support external table for now.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef PLUGSTORAGE_H
+#define PLUGSTORAGE_H
+
+#include "postgres.h"
+#include "postgres_ext.h"
+#include "fmgr.h"
+#include "nodes/pg_list.h"
+#include "nodes/plannodes.h"
+#include "nodes/execnodes.h"
+#include "access/sdir.h"
+#include "access/relscan.h"
+#include "access/extprotocol.h"
+#include "access/tupdesc.h"
+#include "access/fileam.h"
+#include "utils/relcache.h"
+#include "executor/tuptable.h"
+
+/* From src/include/access/fileam.h */
+extern char *getExtTblFormatterTypeInFmtOptsStr(char *fmtStr);
+extern char *getExtTblFormatterTypeInFmtOptsList(List *fmtOpts);
+
+
+/*
+ * ExternalTableType
+ *
+ * enum for different types of external tables.
+ *
+ * The different types of external tables can be combinations of different
+ * protocols and formats. To be specific:
+ * {GPFDIST, GPFDISTS, HTTP, COMMAND, HDFS} X {TEXT, CSV, ORC}
+ *
+ * NOTE:
+ *
+ * The GENERIC external table type is used to simplify the call back
+ * implementation for different combination of external table formats
+ * and protocols. It need to be improved so that each external table
+ * type should get its access method with minimum cost during runtime.
+ *
+ * The fact is that for custom protocol (HDFS), the format types for
+ * TEXT, CSV, and ORC external tables are all 'b' in pg_exttable catalog
+ * table. The formatter information is stored in format options which is
+ * a list strings. Thus, during read and write access of these tables,
+ * there will be performance issue if we get the format type and access
+ * method for them for each tuple.
+ *
+ */
+typedef enum
+{
+ ExternalTableType_GENERIC, /* GENERIC external table format and protocol */
+ ExternalTableType_TEXT, /* TEXT format with gpfdist(s), http, command protocol */
+ ExternalTableType_CSV, /* CSV format with gpfdist(s), http, command protocol */
+ ExternalTableType_PLUG, /* Pluggable format with hdfs protocol, i.e., ORC */
+ ExternalTableType_Invalid
+} ExternalTableType;
+
+/*
+ * PlugStorageValidatorData is the node type that is passed as fmgr "context"
+ * info when a function is called by the Pluggable Storage Framework.
+ */
+typedef struct PlugStorageValidatorData
+{
+ /* see T_PlugStorageValidatorData */
+ NodeTag type;
+
+ /* check if format interfaces are implemented in pg_proc */
+ char *format_name;
+
+ /* check if format options and their values are valid */
+ List *format_opts;
+ char *format_str;
+ char *encoding_name;
+ bool is_writable;
+
+ /* check if format data types are supported in table definition */
+ TupleDesc tuple_desc;
+
+} PlugStorageValidatorData;
+
+typedef struct PlugStorageValidatorData *PlugStorageValidator;
+
+/*
+ * PlugStorageData is used to pass args between hawq and pluggable data formats.
+ */
+typedef struct PlugStorageData
+{
+ NodeTag type; /* see T_PlugStorageData */
+ int ps_table_type;
+ int ps_formatter_type;
+ char *ps_formatter_name;
+ int ps_error_ao_seg_no;
+ Relation ps_relation;
+ PlanState *ps_plan_state;
+ ExternalScan *ps_ext_scan;
+ ScanState *ps_scan_state;
+ ScanDirection ps_scan_direction;
+ FileScanDesc ps_file_scan_desc;
+ ExternalScanState *ps_ext_scan_state;
+ ResultRelSegFileInfo *ps_result_seg_file_info;
+ ExternalInsertDesc ps_ext_insert_desc;
+ ExternalSelectDesc ps_ext_select_desc;
+ bool ps_has_tuple;
+ Oid ps_tuple_oid;
+ TupleTableSlot *ps_tuple_table_slot;
+
+} PlugStorageData;
+
+typedef PlugStorageData *PlugStorage;
+
+
+/*
+ * getExternalTableTypeList
+ * getExternalTableTypeStr
+ *
+ * Return the table type for a given external table
+ *
+ * Currently it is an implementation with performance cost due to the
+ * fact that the format types for TEXT, CSV, and ORC external tables
+ * with customer protocol (HDFS) are all 'b' in pg_exttable catalog
+ * table. It needs to visit the format options to get the table type.
+ */
+void getExternalTableTypeInList(const char formatType,
+ List *formatOptions,
+ int *formatterType,
+ char **formatterName);
+
+void getExternalTableTypeInStr(const char formatType,
+ char *formatOptions,
+ int *formatterType,
+ char **formatterName);
+
+void checkPlugStorageFormatOption(char **opt,
+ const char *key,
+ const char *val,
+ const bool needopt,
+ const int nvalidvals,
+ const char **validvals);
+
+Oid LookupPlugStorageValidatorFunc(char *formatter,
+ char *validator);
+
+void InvokePlugStorageValidationFormatInterfaces(Oid procOid,
+ char *formatName);
+
+void InvokePlugStorageValidationFormatOptions(Oid procOid,
+ List *formatOptions,
+ char *formatStr,
+ bool isWritable);
+
+void InvokePlugStorageValidationFormatEncodings(Oid procOid,
+ char *encodingName);
+
+void InvokePlugStorageValidationFormatDataTypes(Oid procOid,
+ TupleDesc tupDesc);
+
+FileScanDesc InvokePlugStorageFormatBeginScan(FmgrInfo *func,
+ ExternalScan *extScan,
+ ScanState *scanState,
+ Relation relation,
+ int formatterType,
+ char *formatterNam);
+
+ExternalSelectDesc InvokePlugStorageFormatGetNextInit(FmgrInfo *func,
+ PlanState *planState,
+ ExternalScanState *extScanState);
+
+bool InvokePlugStorageFormatGetNext(FmgrInfo *func,
+ FileScanDesc fileScanDesc,
+ ScanDirection scanDirection,
+ ExternalSelectDesc extSelectDesc,
+ ScanState *scanState,
+ TupleTableSlot *tupTableSlot);
+
+void InvokePlugStorageFormatReScan(FmgrInfo *func,
+ FileScanDesc fileScanDesc);
+
+void InvokePlugStorageFormatEndScan(FmgrInfo *func,
+ FileScanDesc fileScanDesc);
+
+void InvokePlugStorageFormatStopScan(FmgrInfo *func,
+ FileScanDesc fileScanDesc);
+
+ExternalInsertDesc InvokePlugStorageFormatInsertInit(FmgrInfo *func,
+ Relation relation,
+ int formatterType,
+ char *formatterName);
+
+Oid InvokePlugStorageFormatInsert(FmgrInfo *func,
+ ExternalInsertDesc extInsertDesc,
+ TupleTableSlot *tupTableSlot);
+
+void InvokePlugStorageFormatInsertFinish(FmgrInfo *func,
+ ExternalInsertDesc extInsertDesc);
+
+#endif /* PLUGSTORAGE_H */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/96c13f7b/src/include/access/plugstorage_utils.h
----------------------------------------------------------------------
diff --git a/src/include/access/plugstorage_utils.h b/src/include/access/plugstorage_utils.h
new file mode 100644
index 0000000..807594e
--- /dev/null
+++ b/src/include/access/plugstorage_utils.h
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*-------------------------------------------------------------------------
+ *
+ * plugstorage_utils.h
+ *
+ * Pluggable storage utility definitions. Support external table for now.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef PLUGSTORAGE_UTILS_H
+#define PLUGSTORAGE_UTILS_H
+
+#include "fmgr.h"
+
+/*
+ * Scan functions for pluggable format
+ */
+typedef struct PlugStorageScanFuncs
+{
+ FmgrInfo *beginscan;
+ FmgrInfo *getnext_init;
+ FmgrInfo *getnext;
+ FmgrInfo *rescan;
+ FmgrInfo *endscan;
+ FmgrInfo *stopscan;
+
+} PlugStorageScanFuncs;
+
+/*
+ * Insert functions for pluggable format
+ */
+typedef struct PlugStorageInsertFuncs
+{
+ FmgrInfo *insert_init;
+ FmgrInfo *insert;
+ FmgrInfo *insert_finish;
+
+} PlugStorageInsertFuncs;
+
+#endif /* PLUGSTORAGE_UTILS_H */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/96c13f7b/src/include/nodes/nodes.h
----------------------------------------------------------------------
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index 3c85f16..b460443 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -512,9 +512,12 @@ typedef enum NodeTag
T_FormatterData, /* in access/formatter.h */
T_ExtProtocolData, /* in access/extprotocol.h */
T_ExtProtocolValidatorData, /* in access/extprotocol.h */
+ T_PlugStorageData, /* in access/plugstorage.h */
+ T_PlugStorageValidatorData, /* in access/plugstorage.h */
T_FileSystemFunctionData, /* in storage/filesystem.h */
T_PartitionConstraints, /* in executor/nodePartitionSelector.h */
T_SelectedParts, /* in executor/nodePartitionSelector.h */
+ T_ExtProtocolBlockLocationData, /* in access/extprotocol.h */
/* CDB: tags for random other stuff */
T_CdbExplain_StatHdr = 950, /* in cdb/cdbexplain.c */