You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by hu...@apache.org on 2016/07/12 06:50:41 UTC
incubator-hawq git commit: HAWQ-808. Refactor feature test for
external_oid with new framework.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 651ce19d1 -> bab04b57b
HAWQ-808. Refactor feature test for external_oid with new framework.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/bab04b57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/bab04b57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/bab04b57
Branch: refs/heads/master
Commit: bab04b57bfc4737b584595e79f61a7afec523c05
Parents: 651ce19
Author: xunzhang <xu...@gmail.com>
Authored: Tue Jul 12 14:14:08 2016 +0800
Committer: xunzhang <xu...@gmail.com>
Committed: Tue Jul 12 14:23:34 2016 +0800
----------------------------------------------------------------------
.../ExternalSource/ans/external_oid.ans.source | 209 ++++++
.../ExternalSource/data/multi_table.json | 1 +
.../ExternalSource/data/single_table.json | 1 +
src/test/feature/ExternalSource/lib/Makefile | 44 ++
src/test/feature/ExternalSource/lib/function.c | 727 +++++++++++++++++++
.../ExternalSource/sql/external_oid.sql.source | 108 +++
.../ExternalSource/test_external_oid.cpp | 35 +
src/test/feature/Makefile | 2 +
src/test/regress/input/external_oid.source | 118 ---
src/test/regress/known_good_schedule | 1 -
src/test/regress/output/external_oid.source | 211 ------
11 files changed, 1127 insertions(+), 330 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/ans/external_oid.ans.source
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/ans/external_oid.ans.source b/src/test/feature/ExternalSource/ans/external_oid.ans.source
new file mode 100644
index 0000000..3f3f3ae
--- /dev/null
+++ b/src/test/feature/ExternalSource/ans/external_oid.ans.source
@@ -0,0 +1,209 @@
+-- --------------------------------------
+-- test first external Oid initialization
+-- --------------------------------------
+-- start_matchsubs
+--
+-- # create a match/subs expression to handle ip addresses that change
+--
+-- m/.*inserted tuple to heap table pg_class \(oid \d+, relname table_xl\).*/
+-- s/oid \d+/oid SOME_OID/
+--
+-- m/.*deleted tuple oid=\d+ from heap table pg_class.*/
+-- s/oid=\d+/oid=OID/
+--
+-- end_matchsubs
+-- Create function that returns the first external Oid boundary
+CREATE OR REPLACE FUNCTION min_external_oid() RETURNS oid
+ AS '@SHARE_LIBRARY_PATH@', 'min_external_oid'
+ LANGUAGE C;
+CREATE FUNCTION
+-- Create function that returns the current external Oid
+CREATE OR REPLACE FUNCTION get_next_external_oid() RETURNS oid
+ AS '@SHARE_LIBRARY_PATH@', 'get_next_external_oid'
+ LANGUAGE C;
+CREATE FUNCTION
+-- Create function that sets the current external Oid
+CREATE OR REPLACE FUNCTION set_next_external_oid(ext_oid oid) RETURNS oid
+ AS '@SHARE_LIBRARY_PATH@', 'set_next_external_oid'
+ LANGUAGE C;
+CREATE FUNCTION
+-- Create function to insert and scan in-memory data to pg_class
+CREATE OR REPLACE FUNCTION load_json_data(filename text) RETURNS text
+ AS '@SHARE_LIBRARY_PATH@', 'load_json_data'
+ LANGUAGE C;
+CREATE FUNCTION
+-- Create function that inserts tuple with given Oid
+CREATE OR REPLACE FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text) RETURNS text
+ AS '@SHARE_LIBRARY_PATH@', 'caql_insert_into_heap_pg_class'
+ LANGUAGE C;
+CREATE FUNCTION
+-- Create function that inserts tuple with given Oid
+CREATE OR REPLACE FUNCTION caql_delete_from_heap_pg_class(relid oid) RETURNS text
+ AS '@SHARE_LIBRARY_PATH@', 'caql_delete_from_heap_pg_class'
+ LANGUAGE C;
+CREATE FUNCTION
+-- --------------------------------------
+-- Test hcat table external oid initialization
+-- --------------------------------------
+-- Boundary should be at FirstExternalObjectId
+--SELECT min_external_oid();
+-- NextExternalObjectId is uninitialized
+SELECT get_next_external_oid();
+ get_next_external_oid
+-----------------------
+ 0
+(1 row)
+
+SELECT load_json_data('@abs_datadir@/single_table.json');
+ load_json_data
+------------------
+ default.mytable
+(1 row)
+
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+ 0
+(1 row)
+
+BEGIN TRANSACTION;
+BEGIN
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+ 0
+(1 row)
+
+-- load default.mytable -> +3 oids
+-- 1 oid for namespace 'default', 1 oid for relation 'mytable', 1 oid for reltype
+SELECT load_json_data('@abs_datadir@/single_table.json');
+ load_json_data
+------------------
+ default.mytable
+(1 row)
+
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+ 3
+(1 row)
+
+-- load db1.ht1, db2.ht1, db2.ht2 -> +8 oids
+-- oids: db1, ht1(db1), db2, ht1(db2), ht2, reltype(ht1, ht1, ht2)
+SELECT load_json_data('@abs_datadir@/multi_table.json');
+ load_json_data
+--------------------------
+ db1.ht1 db2.ht1 db2.ht2
+(1 row)
+
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+ 11
+(1 row)
+
+END TRANSACTION;
+COMMIT
+-- New transaction will reset external Oid start point
+-- Yields the same result as previous transaction
+BEGIN TRANSACTION;
+BEGIN
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+ 0
+(1 row)
+
+SELECT load_json_data('@abs_datadir@/single_table.json');
+ load_json_data
+------------------
+ default.mytable
+(1 row)
+
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+ 3
+(1 row)
+
+SELECT load_json_data('@abs_datadir@/multi_table.json');
+ load_json_data
+--------------------------
+ db1.ht1 db2.ht1 db2.ht2
+(1 row)
+
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+ 11
+(1 row)
+
+END TRANSACTION;
+COMMIT
+-- --------------------------------------
+-- Test external oid rollover
+-- --------------------------------------
+BEGIN TRANSACTION;
+BEGIN
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+ 0
+(1 row)
+
+SELECT set_next_external_oid( oid(min_external_oid()::bigint + (10*power(2,20))::bigint - 8 + 1) ) > 0;
+ ?column?
+----------
+ t
+(1 row)
+
+SELECT load_json_data('@abs_datadir@/multi_table.json');
+ load_json_data
+--------------------------
+ db1.ht1 db2.ht1 db2.ht2
+(1 row)
+
+-- Used up external Oids result in Oid overflow
+SELECT get_next_external_oid();
+ get_next_external_oid
+-----------------------
+ 0
+(1 row)
+
+-- Rollover disallowed!
+SELECT load_json_data('@abs_datadir@/single_table.json');
+psql:/tmp/TestExternalOid_TestExternalOidAll.sql:92: ERROR: number of external objects from HCatalog exceeded 10M during transaction
+HINT: Separate HCatalog queries into different transactions to process.
+END TRANSACTION;
+ROLLBACK
+-- --------------------------------------
+-- Test external Oid boundary
+-- --------------------------------------
+-- Create a tuple with Oid larger than FirstExternalObjectId
+-- Will fail during next session when try to query HCatalog
+-- Because external Oid boundary is violated
+SELECT caql_insert_into_heap_pg_class(min_external_oid()::bigint + 20, 'table_xl');
+ caql_insert_into_heap_pg_class
+--------------------------------------------------------------------------
+ inserted tuple to heap table pg_class (oid 4284481555, relname table_xl)
+(1 row)
+
+-- cleanup
+SELECT caql_delete_from_heap_pg_class(min_external_oid()::bigint + 20);
+ caql_delete_from_heap_pg_class
+-------------------------------------------------------
+ deleted tuple oid=4284481555 from heap table pg_class
+(1 row)
+
+DROP FUNCTION caql_delete_from_heap_pg_class(relid oid);
+DROP FUNCTION
+DROP FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text);
+DROP FUNCTION
+DROP FUNCTION load_json_data(filename text);
+DROP FUNCTION
+DROP FUNCTION get_next_external_oid();
+DROP FUNCTION
+DROP FUNCTION set_next_external_oid(ext_oid oid);
+DROP FUNCTION
+DROP FUNCTION min_external_oid();
+DROP FUNCTION
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/data/multi_table.json
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/data/multi_table.json b/src/test/feature/ExternalSource/data/multi_table.json
new file mode 100644
index 0000000..82e70b0
--- /dev/null
+++ b/src/test/feature/ExternalSource/data/multi_table.json
@@ -0,0 +1 @@
+{"PXFMetadata":[{"item":{"path":"db1","name":"ht1"},"fields":[{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"vc2","type":"varchar","modifiers":["3"],"sourceType":"varchar"}]},{"item":{"path":"db2","name":"ht1"},"fields":[{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"vc2","type":"varchar","modifiers":["3"],"sourceType":"varchar"}]},{"item":{"path":"db2","name":"ht2"},"fields":[{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"vc2","type":"varchar","modifiers":["3"],"sourceType":"varchar"}]}]}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/data/single_table.json
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/data/single_table.json b/src/test/feature/ExternalSource/data/single_table.json
new file mode 100644
index 0000000..b571e5d
--- /dev/null
+++ b/src/test/feature/ExternalSource/data/single_table.json
@@ -0,0 +1 @@
+{"PXFMetadata":[{"item":{"path":"default","name":"mytable"},"fields":[{"name":"s1","type":"text","sourceType":"string"},{"name":"s2","type":"text","sourceType":"string"},{"name":"n1","type":"int4","sourceType":"int"},{"name":"d1","type":"float8","sourceType":"double"},{"name":"dc1","type":"numeric","modifiers":["38","18"],"sourceType":"decimal"},{"name":"tm","type":"timestamp","sourceType":"timestamp"},{"name":"f","type":"float4","sourceType":"float"},{"name":"bg","type":"int8","sourceType":"bigint"},{"name":"b","type":"bool","sourceType":"boolean"},{"name":"tn","type":"int2","sourceType":"tinyint"},{"name":"sml","type":"int2","sourceType":"tinyint"},{"name":"dt","type":"date","sourceType":"date"},{"name":"vc1","type":"varchar","modifiers":["5"],"sourceType":"varchar"},{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"bin","type":"bytea","sourceType":"binary"}]}]}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/lib/Makefile
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/lib/Makefile b/src/test/feature/ExternalSource/lib/Makefile
new file mode 100644
index 0000000..46f6c20
--- /dev/null
+++ b/src/test/feature/ExternalSource/lib/Makefile
@@ -0,0 +1,44 @@
+top_builddir = ../../../../..
+include $(top_builddir)/src/Makefile.global
+
+OS = $(shell uname)
+
+CXX = gcc
+CXXFLAGS = -Wall -O1 -g -std=gnu99 -Wmissing-prototypes -Wpointer-arith -Wendif-labels -Wformat-security -fno-strict-aliasing -fwrapv -fPIC
+
+CC = gcc
+CFLAGS = -Wall -O1 -g -std=gnu99 -Wmissing-prototypes -Wpointer-arith -Wendif-labels -Wformat-security -fno-strict-aliasing -fwrapv -fPIC
+
+CPPFLAGS = -I$(abs_top_srcdir)/src/include
+CPPFLAGS += -I$(abs_top_srcdir)/depends/libhdfs3/build/install$(prefix)/include
+CPPFLAGS += -I$(abs_top_srcdir)/depends/libyarn/build/install$(prefix)/include
+
+LDFLAGS = -L$(libdir)
+LDFLAGS += -L$(abs_top_srcdir)/src/port
+LDFLAGS += -L$(abs_top_builddir)/src/port
+LDFLAGS += -L$(abs_top_srcdir)/depends/libhdfs3/build/install$(prefix)/lib
+LDFLAGS += -L$(abs_top_srcdir)/depends/libyarn/build/install$(prefix)/lib
+
+POSTGRES = $(abs_top_srcdir)/src/backend/postgres
+
+PROG = function.c
+OBJS = function.o
+TARGET = function.so
+
+RM = rm -rf
+
+all: $(TARGET)
+
+$(OBJS): $(PROG)
+ $(CXX) $(CXXFLAGS) $(CPPFLAGS) -c -o $(OBJS) $(PROG)
+
+$(TARGET): $(OBJS)
+ifeq ($(OS),Darwin)
+ $(CXX) $(CXXFLAGS) -bundle $(OBJS) -bundle_loader $(POSTGRES) $(LDFLAGS) -o $@
+else
+ $(CXX) $(CXXFLAGS) -shared $(OBJS) $(LDFLAGS) -Wl,-rpath,'$(abs_top_builddir)/lib' -o $@
+endif
+
+clean:
+ $(RM) *.o
+ $(RM) *.so
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/lib/function.c
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/lib/function.c b/src/test/feature/ExternalSource/lib/function.c
new file mode 100644
index 0000000..4275edc
--- /dev/null
+++ b/src/test/feature/ExternalSource/lib/function.c
@@ -0,0 +1,727 @@
+/*
+ * json_utils.c
+ *
+ * Created on: Mar 5, 2015
+ * Author: antova
+ */
+
+#include "catalog/external/externalmd.h"
+#include "postgres.h"
+#include "access/hd_work_mgr.h"
+#include "funcapi.h"
+#include "catalog/catquery.h"
+#include "catalog/gp_policy.h"
+#include "catalog/namespace.h"
+#include "catalog/pg_database.h"
+#include "catalog/pg_exttable.h"
+#include "catalog/pg_namespace.h"
+#include "cdb/cdbinmemheapam.h"
+#include "nodes/makefuncs.h"
+#include "utils/array.h"
+#include "utils/builtins.h"
+#include "utils/lsyscache.h"
+#include "access/transam.h"
+
+/*
+ * number of output columns for the UDFs for scanning in memory catalog tables
+ */
+#define NUM_COLS 3
+
+#ifdef PG_MODULE_MAGIC
+PG_MODULE_MAGIC;
+#endif
+
+static
+char *read_file(const char *filename);
+
+static
+Oid GetRelationOid(char *tableName);
+
+/* utility functions for loading json files */
+extern Datum load_json_data(PG_FUNCTION_ARGS);
+extern Datum caql_scan_in_memory_pg_namespace(PG_FUNCTION_ARGS);
+extern Datum caql_scan_in_memory_pg_type(PG_FUNCTION_ARGS);
+extern Datum caql_scan_in_memory_gp_distribution_policy(PG_FUNCTION_ARGS);
+extern Datum caql_scan_in_memory_pg_exttable(PG_FUNCTION_ARGS);
+extern Datum caql_scan_in_memory_pg_attribute(PG_FUNCTION_ARGS);
+extern Datum caql_insert_into_heap_pg_class(PG_FUNCTION_ARGS);
+extern Datum caql_delete_from_heap_pg_class(PG_FUNCTION_ARGS);
+extern Datum get_next_external_oid(PG_FUNCTION_ARGS);
+extern Datum set_next_external_oid(PG_FUNCTION_ARGS);
+extern Datum min_external_oid(PG_FUNCTION_ARGS);
+
+PG_FUNCTION_INFO_V1(load_json_data);
+PG_FUNCTION_INFO_V1(caql_scan_in_memory_pg_namespace);
+PG_FUNCTION_INFO_V1(caql_scan_in_memory_pg_type);
+PG_FUNCTION_INFO_V1(caql_scan_in_memory_gp_distribution_policy);
+PG_FUNCTION_INFO_V1(caql_scan_in_memory_pg_exttable);
+PG_FUNCTION_INFO_V1(caql_scan_in_memory_pg_attribute);
+PG_FUNCTION_INFO_V1(caql_insert_into_heap_pg_class);
+PG_FUNCTION_INFO_V1(caql_delete_from_heap_pg_class);
+PG_FUNCTION_INFO_V1(get_next_external_oid);
+PG_FUNCTION_INFO_V1(set_next_external_oid);
+PG_FUNCTION_INFO_V1(min_external_oid);
+
+Datum
+load_json_data(PG_FUNCTION_ARGS)
+{
+ text *filename = PG_GETARG_TEXT_P(0);
+ char *filenameStr = text_to_cstring(filename);
+
+ char *pcbuf = read_file(filenameStr);
+ StringInfoData buf;
+ initStringInfo(&buf);
+ appendStringInfo(&buf, "%s", pcbuf);
+
+ List *items = ParsePxfEntries(&buf, HiveProfileName, HcatalogDbOid);
+ pfree(buf.data);
+
+ StringInfoData tblNames;
+ initStringInfo(&tblNames);
+ ListCell *lc = NULL;
+ foreach (lc, items)
+ {
+ PxfItem *item = (PxfItem *) lfirst(lc);
+ appendStringInfo(&tblNames, "%s.%s ", item->path, item->name);
+ }
+
+ PG_RETURN_TEXT_P(cstring_to_text(tblNames.data));
+}
+
+char *
+read_file(const char *filename)
+{
+ FILE *pf = fopen(filename, "r");
+ if (NULL == pf)
+ {
+ elog(ERROR, "Could not open file %s for reading", filename);
+ }
+
+ StringInfoData strinfo;
+ initStringInfo(&strinfo);
+
+ const int len = 1024;
+ char buf[len];
+
+ while (NULL != fgets(buf, len, pf))
+ {
+ appendStringInfo(&strinfo, "%s", buf);
+ }
+
+ fclose(pf);
+ return strinfo.data;
+}
+
+Datum
+caql_scan_in_memory_pg_namespace(PG_FUNCTION_ARGS)
+{
+ text *namespaceNameText = PG_GETARG_TEXT_P(0);
+ char *namespaceNameStr = text_to_cstring(namespaceNameText);
+
+ FuncCallContext *funcctx;
+ Datum result;
+ MemoryContext oldcontext;
+ TupleDesc tupdesc;
+ HeapTuple restuple;
+ HeapTuple readtup = NULL;
+ cqContext *pcqCtx;
+ StringInfoData buf;
+ initStringInfo(&buf);
+
+ if (SRF_IS_FIRSTCALL())
+ {
+ funcctx = SRF_FIRSTCALL_INIT();
+ /* switch context when allocating stuff to be used in later calls */
+ oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+ tupdesc = CreateTemplateTupleDesc(NUM_COLS, false);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 1, "nspname",
+ TEXTOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "nspdboid",
+ OIDOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "oid",
+ OIDOID, -1, 0);
+ funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+ /* create tuples for pg_namespace table */
+ pcqCtx = caql_beginscan(
+ NULL,
+ cql("SELECT * FROM pg_namespace "
+ " WHERE nspname = :1 and nspdboid = :2",
+ CStringGetDatum((char *) namespaceNameStr), ObjectIdGetDatum(HcatalogDbOid)));
+
+ funcctx->user_fctx = pcqCtx;
+
+ /* return to original context when allocating transient memory */
+ MemoryContextSwitchTo(oldcontext);
+ }
+
+ funcctx = SRF_PERCALL_SETUP();
+ pcqCtx = (cqContext *)funcctx->user_fctx;
+
+ if (NULL != (readtup = caql_getnext(pcqCtx)))
+ {
+ Datum values[NUM_COLS];
+ bool nulls[NUM_COLS];
+
+ Relation relation = RelationIdGetRelation(NamespaceRelationId);
+ TupleDesc tupleDesc = RelationGetDescr(relation);
+
+ char *nspname = DatumGetCString(tuple_getattr(readtup, tupleDesc, Anum_pg_namespace_nspname));
+ text *t = cstring_to_text(nspname);
+
+ values[0] = PointerGetDatum(t);
+ nulls[0] = false;
+
+ values[1] = tuple_getattr(readtup, tupleDesc, Anum_pg_namespace_nspdboid);
+ nulls[1] = false;
+
+ values[2] = ObjectIdGetDatum(HeapTupleGetOid(readtup));
+ nulls[2] = false;
+
+ elog(DEBUG1, "Namespace oid: %d", HeapTupleGetOid(readtup));
+
+ /* build tuple */
+ restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+
+ /* make the tuple into a datum */
+ result = HeapTupleGetDatum(restuple);
+
+ RelationClose(relation);
+ SRF_RETURN_NEXT(funcctx, result);
+ }
+ else
+ {
+ caql_endscan(pcqCtx);
+ SRF_RETURN_DONE(funcctx);
+ }
+}
+
+Datum
+caql_scan_in_memory_pg_type(PG_FUNCTION_ARGS)
+{
+ text *relNameText = PG_GETARG_TEXT_P(0);
+ char *relNameStr = text_to_cstring(relNameText);
+
+ FuncCallContext *funcctx;
+ Datum result;
+ MemoryContext oldcontext;
+ TupleDesc tupdesc;
+ HeapTuple restuple;
+ HeapTuple readtup = NULL;
+ cqContext *pcqCtx;
+ StringInfoData buf;
+ initStringInfo(&buf);
+
+ if (SRF_IS_FIRSTCALL())
+ {
+ funcctx = SRF_FIRSTCALL_INIT();
+ /* switch context when allocating stuff to be used in later calls */
+ oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+ tupdesc = CreateTemplateTupleDesc(NUM_COLS, false);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 1, "typoid",
+ OIDOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "typname",
+ TEXTOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "typnamespace",
+ OIDOID, -1, 0);
+ funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+ /* create tuples for pg_type table */
+ pcqCtx = caql_beginscan(
+ NULL,
+ cql("SELECT * FROM pg_type "
+ " WHERE typname = :1",
+ CStringGetDatum((char *) relNameStr)));
+
+ funcctx->user_fctx = pcqCtx;
+
+ /* return to original context when allocating transient memory */
+ MemoryContextSwitchTo(oldcontext);
+ }
+
+ funcctx = SRF_PERCALL_SETUP();
+ pcqCtx = (cqContext *)funcctx->user_fctx;
+
+ if (NULL != (readtup = caql_getnext(pcqCtx)))
+ {
+ Datum values[NUM_COLS];
+ bool nulls[NUM_COLS];
+
+ Relation relation = RelationIdGetRelation(TypeRelationId);
+ TupleDesc tupleDesc = RelationGetDescr(relation);
+
+ values[0] = ObjectIdGetDatum(HeapTupleGetOid(readtup));
+ nulls[0] = false;
+
+ elog(DEBUG1, "Type oid: %d", HeapTupleGetOid(readtup));
+
+ char *typname = DatumGetCString(tuple_getattr(readtup, tupleDesc, Anum_pg_type_typname));
+ text *t = cstring_to_text(typname);
+
+ values[1] = PointerGetDatum(t);
+ nulls[1] = false;
+
+ values[2] = tuple_getattr(readtup, tupleDesc, Anum_pg_type_typnamespace);
+ nulls[2] = false;
+
+ /* build tuple */
+ restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+
+ /* make the tuple into a datum */
+ result = HeapTupleGetDatum(restuple);
+
+ RelationClose(relation);
+ SRF_RETURN_NEXT(funcctx, result);
+ }
+ else
+ {
+ caql_endscan(pcqCtx);
+ SRF_RETURN_DONE(funcctx);
+ }
+}
+
+Datum
+caql_scan_in_memory_gp_distribution_policy(PG_FUNCTION_ARGS)
+{
+ text *tableName = PG_GETARG_TEXT_P(0);
+ char *tableNameStr = text_to_cstring(tableName);
+ Oid reloid = InvalidOid;
+
+ FuncCallContext *funcctx;
+ Datum result;
+ MemoryContext oldcontext;
+ TupleDesc tupdesc;
+ HeapTuple restuple;
+ HeapTuple pgclasstup = NULL;
+ HeapTuple readtup = NULL;
+ cqContext *pcqCtx;
+ StringInfoData buf;
+ initStringInfo(&buf);
+
+ if (SRF_IS_FIRSTCALL())
+ {
+ funcctx = SRF_FIRSTCALL_INIT();
+ /* switch context when allocating stuff to be used in later calls */
+ oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+ tupdesc = CreateTemplateTupleDesc(NUM_COLS, false);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 1, "tableoid",
+ OIDOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "tablename",
+ TEXTOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "distpolicy",
+ TEXTOID, -1, 0);
+ funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+ /* iterate on pg_class and query
+ * gp_distribution_policy by given oid */
+ pcqCtx = caql_beginscan(
+ NULL,
+ cql("SELECT oid FROM pg_class "
+ " WHERE relname = :1",
+ CStringGetDatum(tableNameStr)));
+
+ funcctx->user_fctx = pcqCtx;
+
+ /* return to original context when allocating transient memory */
+ MemoryContextSwitchTo(oldcontext);
+ }
+
+ funcctx = SRF_PERCALL_SETUP();
+ pcqCtx = (cqContext *)funcctx->user_fctx;
+
+ if (NULL != (pgclasstup = caql_getnext(pcqCtx)))
+ {
+ Datum values[NUM_COLS];
+ bool nulls[NUM_COLS];
+
+ /* create tuples for pg_exttable table */
+ cqContext* pcqCtx1 = caql_beginscan(
+ NULL,
+ cql("SELECT * FROM gp_distribution_policy "
+ " WHERE localoid = :1",
+ ObjectIdGetDatum(HeapTupleGetOid(pgclasstup))));
+
+ Relation relation = RelationIdGetRelation(GpPolicyRelationId);
+ TupleDesc tupleDesc = RelationGetDescr(relation);
+
+ readtup = caql_getnext(pcqCtx1);
+
+ values[0] = ObjectIdGetDatum(reloid);
+ nulls[0] = false;
+
+ text *t1 = cstring_to_text(tableNameStr);
+
+ values[1] = PointerGetDatum(t1);
+ nulls[1] = false;
+
+ text *t2 = NULL;
+ bool isnull;
+ Datum policy = heap_getattr(readtup, Anum_gp_policy_attrnums, tupleDesc, &isnull);
+ if (isnull)
+ {
+ t2 = cstring_to_text("null");
+ }
+ else
+ {
+ /* not tested! */
+ Datum* elems = NULL;
+ int nelems;
+ deconstruct_array(DatumGetArrayTypeP(policy),
+ INT2OID, -1, false, 'i',
+ &elems, NULL, &nelems);
+ Assert(nelems > 0);
+ StringInfoData elems_str;
+ initStringInfo(&elems_str);
+ for (int i = 0; i < nelems; i++)
+ {
+ appendStringInfo(&elems_str, "%d", DatumGetInt16(elems[0]));
+ if (i != nelems)
+ {
+ appendStringInfo(&elems_str, ", ");
+ }
+ }
+ t2 = cstring_to_text(elems_str.data);
+ pfree(elems_str.data);
+ }
+
+ values[2] = PointerGetDatum(t2);
+ nulls[2] = false;
+
+ /* build tuple */
+ restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+
+ /* make the tuple into a datum */
+ result = HeapTupleGetDatum(restuple);
+
+ RelationClose(relation);
+ /* there should be only one value per oid */
+ Assert(NULL == caql_getnext(pcqCtx1)) ;
+ caql_endscan(pcqCtx1);
+
+ SRF_RETURN_NEXT(funcctx, result);
+ }
+ else
+ {
+ caql_endscan(pcqCtx);
+ SRF_RETURN_DONE(funcctx);
+ }
+}
+
+Datum
+caql_scan_in_memory_pg_exttable(PG_FUNCTION_ARGS)
+{
+ text *tableName = PG_GETARG_TEXT_P(0);
+ char *tableNameStr = text_to_cstring(tableName);
+ Oid reloid = InvalidOid;
+
+ FuncCallContext *funcctx;
+ Datum result;
+ MemoryContext oldcontext;
+ TupleDesc tupdesc;
+ HeapTuple restuple;
+ HeapTuple pgclasstup = NULL;
+ HeapTuple readtup = NULL;
+ cqContext *pcqCtx;
+ StringInfoData buf;
+ initStringInfo(&buf);
+
+ if (SRF_IS_FIRSTCALL())
+ {
+ funcctx = SRF_FIRSTCALL_INIT();
+ /* switch context when allocating stuff to be used in later calls */
+ oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+ tupdesc = CreateTemplateTupleDesc(NUM_COLS, false);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 1, "exttableoid",
+ OIDOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "exttablename",
+ TEXTOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "exttablelocation",
+ TEXTOID, -1, 0);
+ funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+ /* iterate on pg_class and query
+ * pg_exttable by given oid */
+ pcqCtx = caql_beginscan(
+ NULL,
+ cql("SELECT oid FROM pg_class "
+ " WHERE relname = :1",
+ CStringGetDatum(tableNameStr)));
+
+ funcctx->user_fctx = pcqCtx;
+
+ /* return to original context when allocating transient memory */
+ MemoryContextSwitchTo(oldcontext);
+ }
+
+ funcctx = SRF_PERCALL_SETUP();
+ pcqCtx = (cqContext *)funcctx->user_fctx;
+
+ if (NULL != (pgclasstup = caql_getnext(pcqCtx)))
+ {
+ Datum values[NUM_COLS];
+ bool nulls[NUM_COLS];
+
+ /* create tuples for pg_exttable table */
+ cqContext* pcqCtx1 = caql_beginscan(
+ NULL,
+ cql("SELECT * FROM pg_exttable "
+ " WHERE reloid = :1",
+ ObjectIdGetDatum(HeapTupleGetOid(pgclasstup))));
+
+ Relation relation = RelationIdGetRelation(ExtTableRelationId);
+ TupleDesc tupleDesc = RelationGetDescr(relation);
+
+ readtup = caql_getnext(pcqCtx1);
+
+ values[0] = ObjectIdGetDatum(reloid);
+ nulls[0] = false;
+
+ text *t1 = cstring_to_text(tableNameStr);
+
+ values[1] = PointerGetDatum(t1);
+ nulls[1] = false;
+
+ Datum locations = tuple_getattr(readtup, tupleDesc, Anum_pg_exttable_location);
+ Datum* elems = NULL;
+ int nelems;
+ deconstruct_array(DatumGetArrayTypeP(locations),
+ TEXTOID, -1, false, 'i',
+ &elems, NULL, &nelems);
+ Assert(nelems > 0);
+ char *loc_str = DatumGetCString(DirectFunctionCall1(textout, elems[0]));
+ text *t2 = cstring_to_text(loc_str);
+
+ values[2] = PointerGetDatum(t2);
+ nulls[2] = false;
+
+ /* build tuple */
+ restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+
+ /* make the tuple into a datum */
+ result = HeapTupleGetDatum(restuple);
+
+ RelationClose(relation);
+ /* there should be only one value per oid */
+ Assert(NULL == caql_getnext(pcqCtx1)) ;
+ caql_endscan(pcqCtx1);
+
+ SRF_RETURN_NEXT(funcctx, result);
+ }
+ else
+ {
+ caql_endscan(pcqCtx);
+ SRF_RETURN_DONE(funcctx);
+ }
+}
+
+Datum
+caql_scan_in_memory_pg_attribute(PG_FUNCTION_ARGS)
+{
+ text *tableNameText = PG_GETARG_TEXT_P(0);
+ char *tableNameStr = text_to_cstring(tableNameText);
+
+ FuncCallContext *funcctx;
+ Datum result;
+ MemoryContext oldcontext;
+ TupleDesc tupdesc;
+ HeapTuple restuple;
+ HeapTuple readtup = NULL;
+ cqContext *pcqCtx;
+ StringInfoData buf;
+ initStringInfo(&buf);
+
+ Oid relid = GetRelationOid(tableNameStr);
+ if (SRF_IS_FIRSTCALL())
+ {
+ funcctx = SRF_FIRSTCALL_INIT();
+ /* switch context when allocating stuff to be used in later calls */
+ oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+ tupdesc = CreateTemplateTupleDesc(NUM_COLS, false);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 1, "attname",
+ TEXTOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 2, "atttype",
+ OIDOID, -1, 0);
+ TupleDescInitEntry(tupdesc, (AttrNumber) 3, "atttypmod",
+ INT4OID, -1, 0);
+ funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+ /* create tuples for pg_attribute table */
+ pcqCtx = caql_beginscan(
+ NULL,
+ cql("SELECT * FROM pg_attribute "
+ " WHERE attrelid = :1",
+ ObjectIdGetDatum(relid)));
+ funcctx->user_fctx = pcqCtx;
+
+ /* return to original context when allocating transient memory */
+ MemoryContextSwitchTo(oldcontext);
+ }
+
+ funcctx = SRF_PERCALL_SETUP();
+ pcqCtx = (cqContext *)funcctx->user_fctx;
+
+ if (NULL != (readtup = caql_getnext(pcqCtx)))
+ {
+ Datum values[NUM_COLS];
+ bool nulls[NUM_COLS];
+
+ Relation relation = RelationIdGetRelation(AttributeRelationId);
+ TupleDesc tupleDesc = RelationGetDescr(relation);
+
+ char *attname = DatumGetCString(tuple_getattr(readtup, tupleDesc, Anum_pg_attribute_attname));
+ text *t = cstring_to_text(attname);
+
+ values[0] = PointerGetDatum(t);
+ nulls[0] = false;
+
+ values[1] = tuple_getattr(readtup, tupleDesc, Anum_pg_attribute_atttypid);
+ nulls[1] = false;
+
+ values[2] = tuple_getattr(readtup, tupleDesc, Anum_pg_attribute_atttypmod);
+ nulls[2] = false;
+
+ /* build tuple */
+ restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+
+ /* make the tuple into a datum */
+ result = HeapTupleGetDatum(restuple);
+
+ RelationClose(relation);
+ SRF_RETURN_NEXT(funcctx, result);
+ }
+ else
+ {
+ caql_endscan(pcqCtx);
+ SRF_RETURN_DONE(funcctx);
+ }
+}
+
+Oid GetRelationOid(char *tableName)
+{
+ cqContext *pcqCtx = caql_beginscan(
+ NULL,
+ cql("SELECT * FROM pg_class "
+ " WHERE relname = :1",
+ CStringGetDatum((char *) tableName)));
+
+ HeapTuple ht = caql_getnext(pcqCtx);
+ Oid result = InvalidOid;
+ if (NULL != ht)
+ {
+ result = HeapTupleGetOid(ht);
+ }
+
+ elog(DEBUG1, "Found relname %s, oid: %d", tableName, result);
+ caql_endscan(pcqCtx);
+ return result;
+}
+
+/*
+ * This function must be used in conjunction with
+ * caql_delete_from_heap_pg_class
+ * and exclusively for testing external Oid setting
+ */
+Datum
+caql_insert_into_heap_pg_class(PG_FUNCTION_ARGS)
+ {
+ Oid relid = PG_GETARG_OID(0);
+ char *tblname = text_to_cstring(PG_GETARG_TEXT_P(1));
+
+ Datum values[Natts_pg_class];
+ bool nulls[Natts_pg_class];
+
+ for (int i = 0; i < Natts_pg_class; i++)
+ {
+ nulls[i] = true;
+ values[i] = (Datum) 0;
+ }
+
+ NameData name;
+ namestrcpy(&name, tblname);
+
+ values[Anum_pg_class_relname - 1] = NameGetDatum(&name);
+ values[Anum_pg_class_relnamespace - 1] = ObjectIdGetDatum((Oid) NSPDBOID_CURRENT);
+ nulls[Anum_pg_class_relname - 1] = false;
+ nulls[Anum_pg_class_relnamespace - 1] = false;
+
+ cqContext *pcqCtx = caql_beginscan(
+ NULL,
+ cql("INSERT INTO pg_class", NULL));
+
+ HeapTuple tup = caql_form_tuple(pcqCtx, values, nulls);
+ HeapTupleSetOid(tup, relid);
+
+ caql_insert(pcqCtx, tup);
+ caql_endscan(pcqCtx);
+
+ StringInfoData buf;
+ initStringInfo(&buf);
+
+ appendStringInfo(&buf, "inserted tuple to heap table pg_class (oid %u, relname %s)", relid, tblname);
+
+ PG_RETURN_TEXT_P(cstring_to_text(buf.data));
+}
+
+/*
+ * This function must be used in conjunction with
+ * caql_insert_into_heap_pg_class
+ * and exclusively for testing external Oid setting
+ */
+Datum
+caql_delete_from_heap_pg_class(PG_FUNCTION_ARGS)
+{
+ Oid relid = PG_GETARG_OID(0);
+
+ cqContext *pcqCtx = caql_beginscan(
+ NULL,
+ cql("SELECT * FROM pg_class "
+ " WHERE oid = :1 "
+ " FOR UPDATE ",
+ ObjectIdGetDatum(relid)));
+
+ HeapTuple tuple = caql_getnext(pcqCtx);
+
+ if (!HeapTupleIsValid(tuple))
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_OBJECT),
+ errmsg("pg_class table relid=%u does not exist", relid)));
+
+ /* Delete the pg_class table entry from the catalog (pg_class) */
+ caql_delete_current(pcqCtx);
+
+ /* Finish up scan and close pg_class catalog */
+ caql_endscan(pcqCtx);
+
+ StringInfoData buf;
+ initStringInfo(&buf);
+
+ appendStringInfo(&buf, "deleted tuple oid=%u from heap table pg_class", relid);
+
+ PG_RETURN_TEXT_P(cstring_to_text(buf.data));
+}
+
+Datum
+get_next_external_oid(PG_FUNCTION_ARGS)
+{
+ PG_RETURN_OID(GetCurrentExternalObjectId());
+}
+
+Datum
+set_next_external_oid(PG_FUNCTION_ARGS)
+{
+ Oid oid = PG_GETARG_OID(0);
+ SetCurrentExternalObjectId(oid);
+
+ PG_RETURN_OID(GetCurrentExternalObjectId());
+}
+
+Datum
+min_external_oid(PG_FUNCTION_ARGS)
+{
+ PG_RETURN_OID(FirstExternalObjectId);
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/sql/external_oid.sql.source
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/sql/external_oid.sql.source b/src/test/feature/ExternalSource/sql/external_oid.sql.source
new file mode 100644
index 0000000..bded23e
--- /dev/null
+++ b/src/test/feature/ExternalSource/sql/external_oid.sql.source
@@ -0,0 +1,108 @@
+-- --------------------------------------
+-- test first external Oid initialization
+-- --------------------------------------
+
+-- start_matchsubs
+--
+-- # create a match/subs expression to handle ip addresses that change
+--
+-- m/.*inserted tuple to heap table pg_class \(oid \d+, relname table_xl\).*/
+-- s/oid \d+/oid SOME_OID/
+--
+-- m/.*deleted tuple oid=\d+ from heap table pg_class.*/
+-- s/oid=\d+/oid=OID/
+--
+-- end_matchsubs
+
+-- Create function that returns the first external Oid boundary
+CREATE OR REPLACE FUNCTION min_external_oid() RETURNS oid
+ AS '@SHARE_LIBRARY_PATH@', 'min_external_oid'
+ LANGUAGE C;
+
+-- Create function that returns the current external Oid
+CREATE OR REPLACE FUNCTION get_next_external_oid() RETURNS oid
+ AS '@SHARE_LIBRARY_PATH@', 'get_next_external_oid'
+ LANGUAGE C;
+
+-- Create function that sets the current external Oid
+CREATE OR REPLACE FUNCTION set_next_external_oid(ext_oid oid) RETURNS oid
+ AS '@SHARE_LIBRARY_PATH@', 'set_next_external_oid'
+ LANGUAGE C;
+
+-- Create function to insert and scan in-memory data to pg_class
+CREATE OR REPLACE FUNCTION load_json_data(filename text) RETURNS text
+ AS '@SHARE_LIBRARY_PATH@', 'load_json_data'
+ LANGUAGE C;
+
+-- Create function that inserts tuple with given Oid
+CREATE OR REPLACE FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text) RETURNS text
+ AS '@SHARE_LIBRARY_PATH@', 'caql_insert_into_heap_pg_class'
+ LANGUAGE C;
+
+-- Create function that inserts tuple with given Oid
+CREATE OR REPLACE FUNCTION caql_delete_from_heap_pg_class(relid oid) RETURNS text
+ AS '@SHARE_LIBRARY_PATH@', 'caql_delete_from_heap_pg_class'
+ LANGUAGE C;
+
+-- --------------------------------------
+-- Test hcat table external oid initialization
+-- --------------------------------------
+-- Boundary should be at FirstExternalObjectId
+--SELECT min_external_oid();
+-- NextExternalObjectId is uninitialized
+SELECT get_next_external_oid();
+SELECT load_json_data('@abs_datadir@/single_table.json');
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+
+BEGIN TRANSACTION;
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+-- load default.mytable -> +3 oids
+-- 1 oid for namespace 'default', 1 oid for relation 'mytable', 1 oid for reltype
+SELECT load_json_data('@abs_datadir@/single_table.json');
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+-- load db1.ht1, db2.ht1, db2.ht2 -> +8 oids
+-- oids: db1, ht1(db1), db2, ht1(db2), ht2, reltype(ht1, ht1, ht2)
+SELECT load_json_data('@abs_datadir@/multi_table.json');
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+END TRANSACTION;
+
+-- New transaction will reset external Oid start point
+-- Yields the same result as previous transaction
+BEGIN TRANSACTION;
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+SELECT load_json_data('@abs_datadir@/single_table.json');
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+SELECT load_json_data('@abs_datadir@/multi_table.json');
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+END TRANSACTION;
+
+-- --------------------------------------
+-- Test external oid rollover
+-- --------------------------------------
+BEGIN TRANSACTION;
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+SELECT set_next_external_oid( oid(min_external_oid()::bigint + (10*power(2,20))::bigint - 8 + 1) ) > 0;
+SELECT load_json_data('@abs_datadir@/multi_table.json');
+-- Used up external Oids result in Oid overflow
+SELECT get_next_external_oid();
+-- Rollover disallowed!
+SELECT load_json_data('@abs_datadir@/single_table.json');
+END TRANSACTION;
+
+-- --------------------------------------
+-- Test external Oid boundary
+-- --------------------------------------
+-- Create a tuple with Oid larger than FirstExternalObjectId
+-- Will fail during next session when try to query HCatalog
+-- Because external Oid boundary is violated
+SELECT caql_insert_into_heap_pg_class(min_external_oid()::bigint + 20, 'table_xl');
+
+-- cleanup
+SELECT caql_delete_from_heap_pg_class(min_external_oid()::bigint + 20);
+
+DROP FUNCTION caql_delete_from_heap_pg_class(relid oid);
+DROP FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text);
+DROP FUNCTION load_json_data(filename text);
+DROP FUNCTION get_next_external_oid();
+DROP FUNCTION set_next_external_oid(ext_oid oid);
+DROP FUNCTION min_external_oid();
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/test_external_oid.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/test_external_oid.cpp b/src/test/feature/ExternalSource/test_external_oid.cpp
new file mode 100644
index 0000000..3326514
--- /dev/null
+++ b/src/test/feature/ExternalSource/test_external_oid.cpp
@@ -0,0 +1,35 @@
+#include <string>
+#include <iostream>
+#include "gtest/gtest.h"
+
+#include "lib/sql_util.h"
+#include "lib/file_replace.h"
+
+using hawq::test::SQLUtility;
+using hawq::test::FileReplace;
+
+class TestExternalOid : public ::testing::Test {
+ public:
+ TestExternalOid() {}
+ ~TestExternalOid() {}
+};
+
+TEST_F(TestExternalOid, TestExternalOidAll) {
+ SQLUtility util;
+ FileReplace frep;
+ auto test_root = util.getTestRootPath();
+ std::cout << test_root << std::endl;
+
+ std::unordered_map<std::string, std::string> D;
+ D["@SHARE_LIBRARY_PATH@"] = test_root + "/ExternalSource/lib/function.so";
+ D["@abs_datadir@"] = test_root + "/ExternalSource/data";
+ frep.replace(test_root + "/ExternalSource/sql/external_oid.sql.source",
+ test_root + "/ExternalSource/sql/external_oid.sql",
+ D);
+ frep.replace(test_root + "/ExternalSource/ans/external_oid.ans.source",
+ test_root + "/ExternalSource/ans/external_oid.ans",
+ D);
+
+ util.execSQLFile("ExternalSource/sql/external_oid.sql",
+ "ExternalSource/ans/external_oid.ans");
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/Makefile
----------------------------------------------------------------------
diff --git a/src/test/feature/Makefile b/src/test/feature/Makefile
index adc6acc..f393a76 100644
--- a/src/test/feature/Makefile
+++ b/src/test/feature/Makefile
@@ -27,9 +27,11 @@ all: $(OBJS) sharelib
sharelib:
cd UDF/lib || exit 1; $(MAKE) || exit 2
+ cd ExternalSource/lib || exit 1; $(MAKE) || exit 2
sharelibclean:
cd UDF/lib || exit 1; $(RM) *.o *.so || exit 2
+ cd ExternalSource/lib || exit 1; $(MAKE) || exit 2
doc:
doxygen doxygen_template
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/regress/input/external_oid.source
----------------------------------------------------------------------
diff --git a/src/test/regress/input/external_oid.source b/src/test/regress/input/external_oid.source
deleted file mode 100644
index bd4724a..0000000
--- a/src/test/regress/input/external_oid.source
+++ /dev/null
@@ -1,118 +0,0 @@
--- --------------------------------------
--- test first external Oid initialization
--- --------------------------------------
-
--- start_matchsubs
---
--- # create a match/subs expression to handle ip addresses that change
---
--- m/.*inserted tuple to heap table pg_class \(oid \d+, relname table_xl\).*/
--- s/oid \d+/oid SOME_OID/
---
--- m/.*deleted tuple oid=\d+ from heap table pg_class.*/
--- s/oid=\d+/oid=OID/
---
--- end_matchsubs
-
--- Create function that returns the first external Oid boundary
-CREATE OR REPLACE FUNCTION min_external_oid() RETURNS oid
- AS '@abs_builddir@/regress@DLSUFFIX@', 'min_external_oid'
- LANGUAGE C;
-
--- Create function that returns the current external Oid
-CREATE OR REPLACE FUNCTION get_next_external_oid() RETURNS oid
- AS '@abs_builddir@/regress@DLSUFFIX@', 'get_next_external_oid'
- LANGUAGE C;
-
--- Create function that sets the current external Oid
-CREATE OR REPLACE FUNCTION set_next_external_oid(ext_oid oid) RETURNS oid
- AS '@abs_builddir@/regress@DLSUFFIX@', 'set_next_external_oid'
- LANGUAGE C;
-
--- Create function to insert and scan in-memory data to pg_class
-CREATE OR REPLACE FUNCTION load_json_data(filename text) RETURNS text
- AS '@abs_builddir@/regress@DLSUFFIX@', 'load_json_data'
- LANGUAGE C;
-
--- Create function that inserts tuple with given Oid
-CREATE OR REPLACE FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text) RETURNS text
- AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_insert_into_heap_pg_class'
- LANGUAGE C;
-
--- Create function that inserts tuple with given Oid
-CREATE OR REPLACE FUNCTION caql_delete_from_heap_pg_class(relid oid) RETURNS text
- AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_delete_from_heap_pg_class'
- LANGUAGE C;
-
--- --------------------------------------
--- Test hcat table external oid initialization
--- --------------------------------------
--- Boundary should be at FirstExternalObjectId
---SELECT min_external_oid();
--- NextExternalObjectId is uninitialized
-SELECT get_next_external_oid();
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
-
-BEGIN TRANSACTION;
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
--- load default.mytable -> +3 oids
--- 1 oid for namespace 'default', 1 oid for relation 'mytable', 1 oid for reltype
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
--- load db1.ht1, db2.ht1, db2.ht2 -> +8 oids
--- oids: db1, ht1(db1), db2, ht1(db2), ht2, reltype(ht1, ht1, ht2)
-SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
-END TRANSACTION;
-
--- New transaction will reset external Oid start point
--- Yields the same result as previous transaction
-BEGIN TRANSACTION;
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
-SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
-END TRANSACTION;
-
--- --------------------------------------
--- Test external oid rollover
--- --------------------------------------
-BEGIN TRANSACTION;
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
-SELECT set_next_external_oid( oid(min_external_oid()::bigint + (10*power(2,20))::bigint - 8 + 1) ) > 0;
-SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
--- Used up external Oids result in Oid overflow
-SELECT get_next_external_oid();
--- Rollover disallowed!
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-END TRANSACTION;
-
--- --------------------------------------
--- Test external Oid boundary
--- --------------------------------------
--- Create a tuple with Oid larger than FirstExternalObjectId
--- Will fail during next session when try to query HCatalog
--- Because external Oid boundary is violated
-SELECT caql_insert_into_heap_pg_class(min_external_oid()::bigint + 20, 'table_xl');
-
--- Fresh session
-\c
--- NextExternalObjectId is uninitialized
-SELECT get_next_external_oid();
--- Should fail
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
--- NextExternalObjectId is still uninitialized
-SELECT get_next_external_oid();
-
--- cleanup
-SELECT caql_delete_from_heap_pg_class(min_external_oid()::bigint + 20);
-
-DROP FUNCTION caql_delete_from_heap_pg_class(relid oid);
-DROP FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text);
-DROP FUNCTION load_json_data(filename text);
-DROP FUNCTION get_next_external_oid();
-DROP FUNCTION set_next_external_oid(ext_oid oid);
-DROP FUNCTION min_external_oid();
-
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/regress/known_good_schedule
----------------------------------------------------------------------
diff --git a/src/test/regress/known_good_schedule b/src/test/regress/known_good_schedule
index 0cd48f3..1f95e49 100755
--- a/src/test/regress/known_good_schedule
+++ b/src/test/regress/known_good_schedule
@@ -202,5 +202,4 @@ ignore: co_disabled
test: caqlinmem
test: hcatalog_lookup
test: json_load
-test: external_oid
test: validator_function
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/regress/output/external_oid.source
----------------------------------------------------------------------
diff --git a/src/test/regress/output/external_oid.source b/src/test/regress/output/external_oid.source
deleted file mode 100644
index d0a3730..0000000
--- a/src/test/regress/output/external_oid.source
+++ /dev/null
@@ -1,211 +0,0 @@
--- --------------------------------------
--- test first external Oid initialization
--- --------------------------------------
--- start_matchsubs
---
--- # create a match/subs expression to handle ip addresses that change
---
--- m/.*inserted tuple to heap table pg_class \(oid \d+, relname table_xl\).*/
--- s/oid \d+/oid SOME_OID/
---
--- m/.*deleted tuple oid=\d+ from heap table pg_class.*/
--- s/oid=\d+/oid=OID/
---
--- end_matchsubs
--- Create function that returns the first external Oid boundary
-CREATE OR REPLACE FUNCTION min_external_oid() RETURNS oid
- AS '@abs_builddir@/regress@DLSUFFIX@', 'min_external_oid'
- LANGUAGE C;
--- Create function that returns the current external Oid
-CREATE OR REPLACE FUNCTION get_next_external_oid() RETURNS oid
- AS '@abs_builddir@/regress@DLSUFFIX@', 'get_next_external_oid'
- LANGUAGE C;
--- Create function that sets the current external Oid
-CREATE OR REPLACE FUNCTION set_next_external_oid(ext_oid oid) RETURNS oid
- AS '@abs_builddir@/regress@DLSUFFIX@', 'set_next_external_oid'
- LANGUAGE C;
--- Create function to insert and scan in-memory data to pg_class
-CREATE OR REPLACE FUNCTION load_json_data(filename text) RETURNS text
- AS '@abs_builddir@/regress@DLSUFFIX@', 'load_json_data'
- LANGUAGE C;
--- Create function that inserts tuple with given Oid
-CREATE OR REPLACE FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text) RETURNS text
- AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_insert_into_heap_pg_class'
- LANGUAGE C;
--- Create function that inserts tuple with given Oid
-CREATE OR REPLACE FUNCTION caql_delete_from_heap_pg_class(relid oid) RETURNS text
- AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_delete_from_heap_pg_class'
- LANGUAGE C;
--- --------------------------------------
--- Test hcat table external oid initialization
--- --------------------------------------
--- Boundary should be at FirstExternalObjectId
---SELECT min_external_oid();
--- NextExternalObjectId is uninitialized
-SELECT get_next_external_oid();
- get_next_external_oid
------------------------
- 0
-(1 row)
-
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
- load_json_data
-------------------
- default.mytable
-(1 row)
-
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column?
-----------
- 0
-(1 row)
-
-BEGIN TRANSACTION;
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column?
-----------
- 0
-(1 row)
-
--- load default.mytable -> +3 oids
--- 1 oid for namespace 'default', 1 oid for relation 'mytable', 1 oid for reltype
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
- load_json_data
-------------------
- default.mytable
-(1 row)
-
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column?
-----------
- 3
-(1 row)
-
--- load db1.ht1, db2.ht1, db2.ht2 -> +8 oids
--- oids: db1, ht1(db1), db2, ht1(db2), ht2, reltype(ht1, ht1, ht2)
-SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
- load_json_data
---------------------------
- db1.ht1 db2.ht1 db2.ht2
-(1 row)
-
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column?
-----------
- 11
-(1 row)
-
-END TRANSACTION;
--- New transaction will reset external Oid start point
--- Yields the same result as previous transaction
-BEGIN TRANSACTION;
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column?
-----------
- 0
-(1 row)
-
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
- load_json_data
-------------------
- default.mytable
-(1 row)
-
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column?
-----------
- 3
-(1 row)
-
-SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
- load_json_data
---------------------------
- db1.ht1 db2.ht1 db2.ht2
-(1 row)
-
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column?
-----------
- 11
-(1 row)
-
-END TRANSACTION;
--- --------------------------------------
--- Test external oid rollover
--- --------------------------------------
-BEGIN TRANSACTION;
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column?
-----------
- 0
-(1 row)
-
-SELECT set_next_external_oid( oid(min_external_oid()::bigint + (10*power(2,20))::bigint - 8 + 1) ) > 0;
- ?column?
-----------
- t
-(1 row)
-
-SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
- load_json_data
---------------------------
- db1.ht1 db2.ht1 db2.ht2
-(1 row)
-
--- Used up external Oids result in Oid overflow
-SELECT get_next_external_oid();
- get_next_external_oid
------------------------
- 0
-(1 row)
-
--- Rollover disallowed!
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-ERROR: number of external objects from HCatalog exceeded 10M during transaction
-HINT: Separate HCatalog queries into different transactions to process.
-END TRANSACTION;
--- --------------------------------------
--- Test external Oid boundary
--- --------------------------------------
--- Create a tuple with Oid larger than FirstExternalObjectId
--- Will fail during next session when try to query HCatalog
--- Because external Oid boundary is violated
-SELECT caql_insert_into_heap_pg_class(min_external_oid()::bigint + 20, 'table_xl');
- caql_insert_into_heap_pg_class
---------------------------------------------------------------------------
- inserted tuple to heap table pg_class (oid 4293918750, relname table_xl)
-(1 row)
-
--- Fresh session
-\c
--- NextExternalObjectId is uninitialized
-SELECT get_next_external_oid();
- get_next_external_oid
------------------------
- 0
-(1 row)
-
--- Should fail
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-ERROR: database does not have enough available Oids to support HCatalog queries
-HINT: Database VACUUM may recycle unused Oids.
--- NextExternalObjectId is still uninitialized
-SELECT get_next_external_oid();
- get_next_external_oid
------------------------
- 0
-(1 row)
-
--- cleanup
-SELECT caql_delete_from_heap_pg_class(min_external_oid()::bigint + 20);
- caql_delete_from_heap_pg_class
--------------------------------------------------------
- deleted tuple oid=4293918750 from heap table pg_class
-(1 row)
-
-DROP FUNCTION caql_delete_from_heap_pg_class(relid oid);
-DROP FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text);
-DROP FUNCTION load_json_data(filename text);
-DROP FUNCTION get_next_external_oid();
-DROP FUNCTION set_next_external_oid(ext_oid oid);
-DROP FUNCTION min_external_oid();