You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by hu...@apache.org on 2018/08/30 06:29:38 UTC
incubator-hawq git commit: HAWQ-1629. Add ORC format using pluggable
storage framework.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 6f3337c65 -> ae2af74b2
HAWQ-1629. Add ORC format using pluggable storage framework.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ae2af74b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ae2af74b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ae2af74b
Branch: refs/heads/master
Commit: ae2af74b2cf6f6679deca6f26be76bdbf7d02fa6
Parents: 6f3337c
Author: oushu1wangziming1 <wa...@oushu.io>
Authored: Fri Aug 24 10:40:02 2018 +0800
Committer: Ruilong Huo <hu...@163.com>
Committed: Thu Aug 30 14:27:36 2018 +0800
----------------------------------------------------------------------
configure | 36 ++++
configure.in | 9 +
contrib/Makefile | 6 +-
contrib/exthdfs/exthdfs.c | 4 +-
contrib/orc/Makefile | 33 ++++
contrib/orc/README | 5 +
contrib/orc/orc.c | 237 +++++++++++++++++++++++++
contrib/orc/orc_init.sql | 60 +++++++
contrib/orc/orc_install.sql | 70 ++++++++
contrib/orc/orc_uninstall.sql | 36 ++++
src/Makefile.global.in | 1 +
src/backend/access/external/plugstorage.c | 44 +++--
src/backend/catalog/Makefile | 10 ++
src/backend/utils/Makefile | 2 +-
src/backend/utils/cache/lsyscache.c | 12 ++
src/include/access/plugstorage_utils.h | 18 +-
src/include/catalog/pg_exttable.h | 2 +
src/include/utils/lsyscache.h | 1 +
18 files changed, 560 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/configure
----------------------------------------------------------------------
diff --git a/configure b/configure
index b0063d3..de64b1b 100755
--- a/configure
+++ b/configure
@@ -731,6 +731,7 @@ krb_srvtab
with_krb5
with_gssapi
with_pgcrypto
+with_orc
with_r
with_java
with_python
@@ -860,6 +861,7 @@ with_python
with_java
with_r
with_pgcrypto
+with_orc
with_gssapi
with_krb5
with_krb_srvnam
@@ -5835,6 +5837,40 @@ $as_echo "$with_pgcrypto" >&6; }
#
+# Optionally build with orc in pluggable storage framework.
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build orc modules" >&5
+$as_echo_n "checking whether to build orc modules... " >&6; }
+
+pgac_args="$pgac_args with_orc"
+
+
+# Check whether --with-orc was given.
+if test "${with_orc+set}" = set; then :
+ withval=$with_orc;
+ case $withval in
+ yes)
+ :
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-orc option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_orc=yes
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_orc" >&5
+$as_echo "$with_orc" >&6; }
+
+
+#
# GSSAPI
#
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with GSSAPI support" >&5
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/configure.in
----------------------------------------------------------------------
diff --git a/configure.in b/configure.in
index b2b6af4..7d1825c 100644
--- a/configure.in
+++ b/configure.in
@@ -723,6 +723,15 @@ AC_MSG_RESULT([$with_pgcrypto])
AC_SUBST(with_pgcrypto)
#
+# Optionally build with orc format in pluggable storage framework.
+#
+AC_MSG_CHECKING([whether to build orc modules])
+PGAC_ARG_BOOL(with, orc, yes, [ --with-orc build with orc format])
+AC_MSG_RESULT([$with_orc])
+AC_SUBST(with_orc)
+#
+
+#
# GSSAPI
#
AC_MSG_CHECKING([whether to build with GSSAPI support])
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/Makefile
----------------------------------------------------------------------
diff --git a/contrib/Makefile b/contrib/Makefile
index e5daff9..a5c87d4 100644
--- a/contrib/Makefile
+++ b/contrib/Makefile
@@ -10,12 +10,16 @@ WANTED_DIRS = \
gp_cancel_query \
formatter_fixedwidth \
exthdfs\
- hawq-hadoop
+ hawq-hadoop\
ifeq ($(with_pgcrypto), yes)
WANTED_DIRS += pgcrypto
endif
+ifeq ($(with_orc), yes)
+WANTED_DIRS += orc
+endif
+
# Missing:
# start-scripts \ (does not have a makefile)
# xml2 \ (requires libxml installed)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/exthdfs/exthdfs.c
----------------------------------------------------------------------
diff --git a/contrib/exthdfs/exthdfs.c b/contrib/exthdfs/exthdfs.c
index 1378734..548d784 100644
--- a/contrib/exthdfs/exthdfs.c
+++ b/contrib/exthdfs/exthdfs.c
@@ -251,13 +251,13 @@ Datum hdfsprotocol_validate(PG_FUNCTION_ARGS)
}
}
}
- /*if(1)
+ if(1)
{
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
errmsg("hdfsprotocol_validate : "
"no formatter is supported for external hdfs")));
- }*/
+ }
if (!isCsv && !isText && !isOrc)
{
ereport(ERROR,
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/orc/Makefile
----------------------------------------------------------------------
diff --git a/contrib/orc/Makefile b/contrib/orc/Makefile
new file mode 100644
index 0000000..f793f48
--- /dev/null
+++ b/contrib/orc/Makefile
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+MODULE_big = orc
+OBJS = orc.o
+
+PG_CPPFLAGS = -I$(libpq_srcdir)
+PG_LIBS = $(libpq_pgport)
+
+
+ifdef USE_PGXS
+PGXS := $(shell pg_config --pgxs)
+include $(PGXS)
+else
+subdir = contrib/orc
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/orc/README
----------------------------------------------------------------------
diff --git a/contrib/orc/README b/contrib/orc/README
new file mode 100644
index 0000000..0d669d3
--- /dev/null
+++ b/contrib/orc/README
@@ -0,0 +1,5 @@
+1. Compile ORC format in pluggable storage framework
+ $ ./configure --with-orc; make -j8; make install
+
+2. Configure and initialize cluster
+ $ hawq init cluster -a
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/orc/orc.c
----------------------------------------------------------------------
diff --git a/contrib/orc/orc.c b/contrib/orc/orc.c
new file mode 100644
index 0000000..729c729
--- /dev/null
+++ b/contrib/orc/orc.c
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+#include <json-c/json.h>
+
+#include "c.h"
+#include "port.h"
+#include "postgres.h"
+#include "fmgr.h"
+#include "funcapi.h"
+#include "nodes/pg_list.h"
+#include "utils/memutils.h"
+#include "utils/relcache.h"
+#include "utils/uri.h"
+#include "utils/formatting.h"
+#include "utils/lsyscache.h"
+#include "utils/datetime.h"
+#include "mb/pg_wchar.h"
+#include "commands/defrem.h"
+#include "commands/copy.h"
+#include "access/tupdesc.h"
+#include "access/filesplit.h"
+#include "access/plugstorage.h"
+#include "cdb/cdbvars.h"
+#include "catalog/pg_exttable.h"
+#include "catalog/namespace.h"
+#include "postmaster/identity.h"
+#include "nodes/makefuncs.h"
+#include "nodes/plannodes.h"
+#include "utils/uri.h"
+
+
+#define ORC_TIMESTAMP_EPOCH_JDATE 2457024 /* == date2j(2015, 1, 1) */
+#define MAX_ORC_ARRAY_DIMS 10000
+
+/* Do the module magic dance */
+PG_MODULE_MAGIC;
+
+/* Validators for pluggable storage format ORC */
+PG_FUNCTION_INFO_V1(orc_validate_interfaces);
+PG_FUNCTION_INFO_V1(orc_validate_options);
+PG_FUNCTION_INFO_V1(orc_validate_encodings);
+PG_FUNCTION_INFO_V1(orc_validate_datatypes);
+
+/* Accessors for pluggable storage format ORC */
+PG_FUNCTION_INFO_V1(orc_beginscan);
+PG_FUNCTION_INFO_V1(orc_getnext_init);
+PG_FUNCTION_INFO_V1(orc_getnext);
+PG_FUNCTION_INFO_V1(orc_rescan);
+PG_FUNCTION_INFO_V1(orc_endscan);
+PG_FUNCTION_INFO_V1(orc_stopscan);
+PG_FUNCTION_INFO_V1(orc_insert_init);
+PG_FUNCTION_INFO_V1(orc_insert);
+PG_FUNCTION_INFO_V1(orc_insert_finish);
+
+/* Definitions of validators for pluggable storage format ORC */
+Datum orc_validate_interfaces(PG_FUNCTION_ARGS);
+Datum orc_validate_options(PG_FUNCTION_ARGS);
+Datum orc_validate_encodings(PG_FUNCTION_ARGS);
+Datum orc_validate_datatypes(PG_FUNCTION_ARGS);
+
+/* Definitions of accessors for pluggable storage format ORC */
+Datum orc_beginscan(PG_FUNCTION_ARGS);
+Datum orc_getnext_init(PG_FUNCTION_ARGS);
+Datum orc_getnext(PG_FUNCTION_ARGS);
+Datum orc_rescan(PG_FUNCTION_ARGS);
+Datum orc_endscan(PG_FUNCTION_ARGS);
+Datum orc_stopscan(PG_FUNCTION_ARGS);
+Datum orc_insert_init(PG_FUNCTION_ARGS);
+Datum orc_insert(PG_FUNCTION_ARGS);
+Datum orc_insert_finish(PG_FUNCTION_ARGS);
+
+
+Datum orc_validate_interfaces(PG_FUNCTION_ARGS)
+{
+ elog(ERROR, "Funtion orc_validate_interfaces has not be completed, please fill it");
+ PG_RETURN_VOID();
+}
+
+/*
+ * void
+ * orc_validate_options(List *formatOptions,
+ * char *formatStr,
+ * bool isWritable)
+ */
+Datum orc_validate_options(PG_FUNCTION_ARGS)
+{
+
+ elog(ERROR, "Funtion orc_validate_options has not be completed, please fill it");
+ PG_RETURN_VOID();
+}
+
+/*
+ * void
+ * orc_validate_encodings(char *encodingName)
+ */
+Datum orc_validate_encodings(PG_FUNCTION_ARGS)
+{
+ elog(ERROR, "Funtion orc_validate_encodings has not be completed, please fill it");
+ PG_RETURN_VOID();
+}
+
+/*
+ * void
+ * orc_validate_datatypes(TupleDesc tupDesc)
+ */
+Datum orc_validate_datatypes(PG_FUNCTION_ARGS)
+{
+
+ elog(ERROR, "Funtion orc_validate_datatypes has not be completed, please fill it");
+ PG_RETURN_VOID();
+}
+
+/*
+ * FileScanDesc
+ * orc_beginscan(ExternalScan *extScan,
+ * ScanState *scanState,
+ * Relation relation,
+ * int formatterType,
+ * char *formatterName)
+ */
+Datum orc_beginscan(PG_FUNCTION_ARGS)
+{
+
+ elog(ERROR, "Funtion orc_beginscan has not be completed, please fill it");
+ PG_RETURN_POINTER(NULL);
+}
+
+/*
+ * ExternalSelectDesc
+ * orc_getnext_init(PlanState *planState,
+ * ExternalScanState *extScanState)
+ */
+Datum orc_getnext_init(PG_FUNCTION_ARGS)
+{
+
+ elog(ERROR, "Funtion orc_getnext_init has not be completed, please fill it");
+ PG_RETURN_POINTER(NULL);
+}
+
+/*
+ * bool
+ * orc_getnext(FileScanDesc fileScanDesc,
+ * ScanDirection direction,
+ * ExternalSelectDesc extSelectDesc,
+ * ScanState *scanState,
+ * TupleTableSlot *tupTableSlot)
+ */
+Datum orc_getnext(PG_FUNCTION_ARGS)
+{
+ elog(ERROR, "Funtion orc_getnext has not be completed, please fill it");
+ PG_RETURN_VOID();
+}
+
+/*
+ * void
+ * orc_rescan(FileScanDesc scan)
+ */
+Datum orc_rescan(PG_FUNCTION_ARGS)
+{
+ elog(ERROR, "Funtion orc_rescan has not be completed, please fill it");
+ PG_RETURN_VOID();
+}
+
+/*
+ * void
+ * orc_endscan(FileScanDesc scan)
+ */
+Datum orc_endscan(PG_FUNCTION_ARGS)
+{
+
+ elog(ERROR, "Funtion orc_endscan has not be completed, please fill it");
+ PG_RETURN_VOID();
+}
+
+/*
+ * void
+ * orc_stopscan(FileScanDesc scan)
+ */
+Datum orc_stopscan(PG_FUNCTION_ARGS)
+{
+ elog(ERROR, "Funtion orc_stopscan has not be completed, please fill it");
+ PG_RETURN_VOID();
+}
+
+/*
+ * ExternalInsertDesc
+ * orc_insert_init(Relation relation,
+ * int formatterType,
+ * char *formatterName)
+ */
+Datum orc_insert_init(PG_FUNCTION_ARGS)
+{
+
+ elog(ERROR, "Funtion orc_insert_init has not be completed, please fill it");
+ PG_RETURN_POINTER(NULL);
+}
+
+/*
+ * Oid
+ * orc_insert(ExternalInsertDesc extInsertDesc,
+ * TupleTableSlot *tupTableSlot)
+ */
+Datum orc_insert(PG_FUNCTION_ARGS)
+{
+
+ elog(ERROR, "Funtion orc_insert has not be completed, please fill it");
+ PG_RETURN_OID(InvalidOid);
+}
+
+/*
+ * void
+ * orc_insert_finish(ExternalInsertDesc extInsertDesc)
+ */
+Datum orc_insert_finish(PG_FUNCTION_ARGS)
+{
+ elog(ERROR, "Funtion orc_insert_finish has not be completed, please fill it");
+ PG_RETURN_VOID();
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/orc/orc_init.sql
----------------------------------------------------------------------
diff --git a/contrib/orc/orc_init.sql b/contrib/orc/orc_init.sql
new file mode 100644
index 0000000..fa720b7
--- /dev/null
+++ b/contrib/orc/orc_init.sql
@@ -0,0 +1,60 @@
+-- --------------------------------------------------------------------
+--
+-- orc_init.sql
+--
+-- Support ORC format in pluggable storage framework at initialization
+--
+-- --------------------------------------------------------------------
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_interfaces() RETURNS void
+AS '$libdir/orc.so', 'orc_validate_interfaces'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_options() RETURNS void
+AS '$libdir/orc.so', 'orc_validate_options'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_encodings() RETURNS void
+AS '$libdir/orc.so', 'orc_validate_encodings'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_datatypes() RETURNS void
+AS '$libdir/orc.so', 'orc_validate_datatypes'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_beginscan() RETURNS bytea
+AS '$libdir/orc.so', 'orc_beginscan'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_getnext_init() RETURNS bytea
+AS '$libdir/orc.so', 'orc_getnext_init'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_getnext() RETURNS bytea
+AS '$libdir/orc.so', 'orc_getnext'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_rescan() RETURNS void
+AS '$libdir/orc.so', 'orc_rescan'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_endscan() RETURNS void
+AS '$libdir/orc.so', 'orc_endscan'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_stopscan() RETURNS void
+AS '$libdir/orc.so', 'orc_stopscan'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_insert_init() RETURNS bytea
+AS '$libdir/orc.so', 'orc_insert_init'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_insert() RETURNS bytea
+AS '$libdir/orc.so', 'orc_insert'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_insert_finish() RETURNS void
+AS '$libdir/orc.so', 'orc_insert_finish'
+LANGUAGE C STABLE;
+
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/orc/orc_install.sql
----------------------------------------------------------------------
diff --git a/contrib/orc/orc_install.sql b/contrib/orc/orc_install.sql
new file mode 100644
index 0000000..424019e
--- /dev/null
+++ b/contrib/orc/orc_install.sql
@@ -0,0 +1,70 @@
+-- --------------------------------------------------------------------
+--
+-- orc_install.sql
+--
+-- Support ORC format in pluggable storage framework
+--
+-- --------------------------------------------------------------------
+
+SET allow_system_table_mods=ddl;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_interfaces() RETURNS void
+AS '$libdir/orc.so', 'orc_validate_interfaces'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_options() RETURNS void
+AS '$libdir/orc.so', 'orc_validate_options'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_encodings() RETURNS void
+AS '$libdir/orc.so', 'orc_validate_encodings'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_validate_datatypes() RETURNS void
+AS '$libdir/orc.so', 'orc_validate_datatypes'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_beginscan() RETURNS bytea
+AS '$libdir/orc.so', 'orc_beginscan'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_getnext_init() RETURNS bytea
+AS '$libdir/orc.so', 'orc_getnext_init'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_getnext() RETURNS bytea
+AS '$libdir/orc.so', 'orc_getnext'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_rescan() RETURNS void
+AS '$libdir/orc.so', 'orc_rescan'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_endscan() RETURNS void
+AS '$libdir/orc.so', 'orc_endscan'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_stopscan() RETURNS void
+AS '$libdir/orc.so', 'orc_stopscan'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_insert_init() RETURNS bytea
+AS '$libdir/orc.so', 'orc_insert_init'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_insert() RETURNS bytea
+AS '$libdir/orc.so', 'orc_insert'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION pg_catalog.orc_insert_finish() RETURNS void
+AS '$libdir/orc.so', 'orc_insert_finish'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION hdfs_validate() RETURNS void
+AS '$libdir/exthdfs.so', 'hdfsprotocol_validate'
+LANGUAGE C STABLE;
+
+CREATE OR REPLACE FUNCTION hdfs_blocklocation() RETURNS void
+AS '$libdir/exthdfs.so', 'hdfsprotocol_blocklocation'
+LANGUAGE C STABLE;
+
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/contrib/orc/orc_uninstall.sql
----------------------------------------------------------------------
diff --git a/contrib/orc/orc_uninstall.sql b/contrib/orc/orc_uninstall.sql
new file mode 100644
index 0000000..1ad03b4
--- /dev/null
+++ b/contrib/orc/orc_uninstall.sql
@@ -0,0 +1,36 @@
+-- --------------------------------------------------------------------
+--
+-- orc_uninstall.sql
+--
+-- Remove ORC format in pluggable storage framework
+--
+-- --------------------------------------------------------------------
+
+SET allow_system_table_mods=ddl;
+
+DROP FUNCTION IF EXISTS pg_catalog.orc_validate_interfaces();
+
+DROP FUNCTION IF EXISTS pg_catalog.orc_validate_options();
+
+DROP FUNCTION IF EXISTS pg_catalog.orc_validate_encodings();
+
+DROP FUNCTION IF EXISTS pg_catalog.orc_validate_datatypes();
+
+DROP FUNCTION IF EXISTS pg_catalog.orc_beginscan();
+
+DROP FUNCTION IF EXISTS pg_catalog.orc_getnext_init();
+
+DROP FUNCTION IF EXISTS pg_catalog.orc_getnext();
+
+DROP FUNCTION IF EXISTS pg_catalog.orc_rescan();
+
+DROP FUNCTION IF EXISTS pg_catalog.orc_endscan();
+
+DROP FUNCTION IF EXISTS pg_catalog.orc_stopscan();
+
+DROP FUNCTION IF EXISTS pg_catalog.orc_insert_init();
+
+DROP FUNCTION IF EXISTS pg_catalog.orc_insert();
+
+DROP FUNCTION IF EXISTS pg_catalog.orc_insert_finish();
+
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/Makefile.global.in
----------------------------------------------------------------------
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index 9dae4bb..3691d79 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -169,6 +169,7 @@ with_python = @with_python@
with_tcl = @with_tcl@
with_r = @with_r@
with_pgcrypto = @with_pgcrypto@
+with_orc = @with_orc@
with_java = @with_java@
with_openssl = @with_openssl@
with_system_tzdata = @with_system_tzdata@
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/backend/access/external/plugstorage.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/plugstorage.c b/src/backend/access/external/plugstorage.c
index fe7c82a..91cb11c 100644
--- a/src/backend/access/external/plugstorage.c
+++ b/src/backend/access/external/plugstorage.c
@@ -33,7 +33,7 @@
#include "parser/parse_func.h"
/*
- * getExternalTableTypeIn(List/Str)
+ * getExternalTableTypeList
*
* Return the table type for a given external table
*
@@ -59,16 +59,22 @@ void getExternalTableTypeInList(const char formatType,
}
else if (fmttype_is_custom(formatType))
{
- *formatterType = ExternalTableType_CUSTOM;
- *formatterName = NULL;
- }
- else if (fmttype_is_custom(formatType))
- {
Assert(formatOptions);
*formatterName = getExtTblFormatterTypeInFmtOptsList(formatOptions);
- *formatterType = ExternalTableType_PLUG;
+ if (pg_strncasecmp(*formatterName, "text", strlen("text")) == 0)
+ {
+ *formatterType = ExternalTableType_TEXT_CUSTOM;
+ }
+ else if (pg_strncasecmp(*formatterName, "csv", strlen("csv")) == 0)
+ {
+ *formatterType = ExternalTableType_CSV_CUSTOM;
+ }
+ else
+ {
+ *formatterType = ExternalTableType_PLUG;
+ }
}
else
{
@@ -78,9 +84,13 @@ void getExternalTableTypeInList(const char formatType,
}
}
+/*
+ * getExternalTableTypeStr
+ *
+ */
void getExternalTableTypeInStr(const char formatType,
char *formatOptions,
- int *formatterType,
+ int *formatterType,
char **formatterName)
{
if (fmttype_is_text(formatType))
@@ -95,17 +105,23 @@ void getExternalTableTypeInStr(const char formatType,
}
else if (fmttype_is_custom(formatType))
{
- *formatterType = ExternalTableType_CUSTOM;
- *formatterName = NULL;
- }
- else if (fmttype_is_custom(formatType))
- {
Assert(formatOptions);
*formatterName = getExtTblFormatterTypeInFmtOptsStr(formatOptions);
Assert(*formatterName);
- *formatterType = ExternalTableType_PLUG;
+ if (pg_strncasecmp(*formatterName, "text", strlen("text")) == 0)
+ {
+ *formatterType = ExternalTableType_TEXT_CUSTOM;
+ }
+ else if (pg_strncasecmp(*formatterName, "csv", strlen("csv")) == 0)
+ {
+ *formatterType = ExternalTableType_CSV_CUSTOM;
+ }
+ else
+ {
+ *formatterType = ExternalTableType_PLUG;
+ }
}
else
{
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/backend/catalog/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile
index 68bfa2b..f50074a 100644
--- a/src/backend/catalog/Makefile
+++ b/src/backend/catalog/Makefile
@@ -30,6 +30,10 @@ include $(top_srcdir)/src/backend/common.mk
cdb_init.sql: cdb_schema.sql filesystem.sql
cat $(call vpathsearch,cdb_schema.sql) $(call vpathsearch,cdb_external_extensions.sql) $(call vpathsearch,filesystem.sql) > cdb_init.sql
+ifeq ($(with_orc), yes)
+ cat $(srcdir)/../../../contrib/orc/orc_init.sql >> cdb_init.sql
+endif
+
cdb_schema.sql: cdb_schema.in
sed -e 's,@pkglibdir@,$(pkglibdir),g' \
-e 's,@dlsuffix@,$(DLSUFFIX),g' \
@@ -139,6 +143,12 @@ install-data: $(BKIFILES) installdirs
$(INSTALL_DATA) $(call vpathsearch,cdb_init.sql) '$(DESTDIR)$(datadir)/cdb_init.sql'
$(INSTALL_DATA) $(call vpathsearch,cdb_external_extensions.sql) '$(DESTDIR)$(datadir)/cdb_external_extensions.sql'
$(INSTALL_DATA) $(call vpathsearch,filesystem.sql) '$(DESTDIR)$(datadir)/filesystem.sql'
+
+ifeq ($(with_orc), yes)
+ $(INSTALL_DATA) $(srcdir)/../../../contrib/orc/orc_install.sql '$(DESTDIR)$(datadir)/orc_install.sql'
+ $(INSTALL_DATA) $(srcdir)/../../../contrib/orc/orc_uninstall.sql '$(DESTDIR)$(datadir)/orc_uninstall.sql'
+endif
+
$(INSTALL_DATA) $(srcdir)/sql_features.txt '$(DESTDIR)$(datadir)/sql_features.txt'
$(INSTALL_DATA) $(call vpathsearch,gp_toolkit.sql) '$(DESTDIR)$(datadir)/gp_toolkit.sql'
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/backend/utils/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/utils/Makefile b/src/backend/utils/Makefile
index da8a82e..05b02f0 100644
--- a/src/backend/utils/Makefile
+++ b/src/backend/utils/Makefile
@@ -8,7 +8,7 @@ subdir = src/backend/utils
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
-OBJS = fmgrtab.o session_state.o
+OBJS = fmgrtab.o session_state.o
SUBDIRS = adt cache error fmgr hash init mb misc mmgr resowner \
resscheduler sort time gpmon gp workfile_manager
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/backend/utils/cache/lsyscache.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c
index 3ccf847..2bd929f 100644
--- a/src/backend/utils/cache/lsyscache.c
+++ b/src/backend/utils/cache/lsyscache.c
@@ -2553,6 +2553,18 @@ type_is_rowtype(Oid typid)
}
/*
+ * type_is_basetype
+ *
+ * Convenience function to determine whether a type OID represents
+ * a "rowtype" type --- neither invalid & a named base type.
+ */
+bool
+type_is_basetype(Oid typid)
+{
+ return (typid != InvalidOid && get_typtype(typid) == TYPTYPE_BASE);
+}
+
+/*
* get_typ_typrelid
*
* Given the type OID, get the typrelid (InvalidOid if not a complex
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/include/access/plugstorage_utils.h
----------------------------------------------------------------------
diff --git a/src/include/access/plugstorage_utils.h b/src/include/access/plugstorage_utils.h
index e0a397d..77686a0 100644
--- a/src/include/access/plugstorage_utils.h
+++ b/src/include/access/plugstorage_utils.h
@@ -80,14 +80,16 @@ typedef struct PlugStorageInsertFuncs
* method for them for each tuple.
*
*/
-typedef enum
-{
- ExternalTableType_GENERIC, /* GENERIC external table format and protocol */
- ExternalTableType_TEXT, /* TEXT format with gpfdist(s), http, command protocol */
- ExternalTableType_CSV, /* CSV format with gpfdist(s), http, command protocol */
- ExternalTableType_CUSTOM, /* Temporary workaround to support the old framework */
- ExternalTableType_PLUG, /* Pluggable format with hdfs protocol, i.e., ORC */
- ExternalTableType_Invalid
+typedef enum {
+ ExternalTableType_GENERIC, /* GENERIC external table format and protocol */
+ ExternalTableType_TEXT, /* TEXT format with gpfdist(s), http, command protocol
+ */
+ ExternalTableType_CSV, /* CSV format with gpfdist(s), http, command protocol
+ */
+ ExternalTableType_TEXT_CUSTOM, /* TEXT format with hdfs protocol */
+ ExternalTableType_CSV_CUSTOM, /* CSV format with hdfs protocol */
+ ExternalTableType_PLUG, /* Pluggable format with hdfs protocol, i.e., ORC */
+ ExternalTableType_Invalid
} ExternalTableType;
#endif /* PLUGSTORAGE_UTILS_H */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/include/catalog/pg_exttable.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/pg_exttable.h b/src/include/catalog/pg_exttable.h
index 11f053c..d5dd8dc 100644
--- a/src/include/catalog/pg_exttable.h
+++ b/src/include/catalog/pg_exttable.h
@@ -167,6 +167,7 @@ RemoveExtTableEntry(Oid relid);
#define CustomFormatType 'b'
#define TextFormatType 't'
#define CsvFormatType 'c'
+#define OrcFormatType 'o'
#define NonCustomFormatType 'n'
/* PXF formats*/
@@ -176,5 +177,6 @@ RemoveExtTableEntry(Oid relid);
#define fmttype_is_custom(c) (c == CustomFormatType)
#define fmttype_is_text(c) (c == TextFormatType)
#define fmttype_is_csv(c) (c == CsvFormatType)
+#define fmttype_is_orc(c) (c == OrcFormatType)
#endif /* PG_EXTTABLE_H */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ae2af74b/src/include/utils/lsyscache.h
----------------------------------------------------------------------
diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h
index 6ee99be..b3c9828 100644
--- a/src/include/utils/lsyscache.h
+++ b/src/include/utils/lsyscache.h
@@ -125,6 +125,7 @@ extern char get_typstorage(Oid typid);
extern Node *get_typdefault(Oid typid);
extern char get_typtype(Oid typid);
extern bool type_is_rowtype(Oid typid);
+extern bool type_is_basetype(Oid typid);
extern Oid get_typ_typrelid(Oid typid);
extern Oid get_element_type(Oid typid);
extern Oid get_array_type(Oid typid);