You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by hu...@apache.org on 2019/05/22 14:45:58 UTC
[hawq] branch master updated: HAWQ-1720. support create table in
orc format
This is an automated email from the ASF dual-hosted git repository.
huor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hawq.git
The following commit(s) were added to refs/heads/master by this push:
new b935042 HAWQ-1720. support create table in orc format
b935042 is described below
commit b9350428ea06a6dfee946fb54452a304238800c4
Author: tuyu <tu...@oushu.io>
AuthorDate: Wed May 22 16:18:41 2019 +0800
HAWQ-1720. support create table in orc format
---
contrib/exthdfs/Makefile | 2 +-
contrib/exthdfs/exthdfs.c | 49 ++++++++++++++--------
src/backend/catalog/gp_toolkit.sql.in | 12 +++---
src/backend/parser/gram.y | 76 ++++++++++++++++++++++++++++++++++-
4 files changed, 114 insertions(+), 25 deletions(-)
diff --git a/contrib/exthdfs/Makefile b/contrib/exthdfs/Makefile
index e247664..7bf514f 100644
--- a/contrib/exthdfs/Makefile
+++ b/contrib/exthdfs/Makefile
@@ -21,7 +21,7 @@ OBJS = exthdfs.o
PG_CPPFLAGS = -I$(libpq_srcdir)
PG_LIBS = $(libpq_pgport)
-override CFLAGS += -lhdfs3
+override CFLAGS += -lhdfs3 -lstorage
ifdef USE_PGXS
PGXS := $(shell pg_config --pgxs)
diff --git a/contrib/exthdfs/exthdfs.c b/contrib/exthdfs/exthdfs.c
index 70e6510..2c501ec 100644
--- a/contrib/exthdfs/exthdfs.c
+++ b/contrib/exthdfs/exthdfs.c
@@ -22,13 +22,11 @@
#include "common.h"
#include "access/extprotocol.h"
#include "cdb/cdbdatalocality.h"
-#include "storage/fd.h"
-#include "storage/filesystem.h"
+#include "cdb/cdbfilesystemcredential.h"
+#include "cdb/cdbvars.h"
+#include "storage/cwrapper/hdfs-file-system-c.h"
#include "utils/uri.h"
-
-
-
PG_MODULE_MAGIC;
PG_FUNCTION_INFO_V1(hdfsprotocol_blocklocation);
@@ -421,30 +419,47 @@ Datum hdfsprotocol_validate(PG_FUNCTION_ARGS)
}
/* SHOULD ADD LOGIC HERE TO CREATE UNEXISTING PATH */
- if (pvalidator_data->forceCreateDir) {
-
+ if (pvalidator_data->forceCreateDir)
+ {
elog(LOG, "hdfs_validator() forced creating dir");
- /* Create file system instance */
- hdfsFS fs = hdfsConnect(uri->hostname, uri->port);
+ /* Create file system instance */
+ FscHdfsFileSystemC *fs = FscHdfsNewFileSystem(uri->hostname,
+ uri->port);
if (fs == NULL)
{
elog(ERROR, "hdfsprotocol_validate : "
- "failed to create HDFS instance to connect to %s:%d",
- uri->hostname, uri->port);
+ "failed to create HDFS instance to connect to %s:%d",
+ uri->hostname, uri->port);
}
- if (hdfsExists(fs, uri->path) == -1)
+ if (FscHdfsExistPath(fs, uri->path)
+ && FscHdfsGetFileKind(fs, uri->path) == 'F')
elog(ERROR, "hdfsprotocol_validate : "
- "Location \"%s\" is not exist",
- uri->path);
+ "Location \"%s\" is a file, not supported yet. "
+ "Only support directory now",
+ uri->path);
+ if (pvalidator_data->direction == EXT_VALIDATE_WRITE
+ && !FscHdfsExistInsertPath(fs, uri->path))
+ {
+ elog(LOG, "hdfs_validator() to create url %s", uri->path);
+ FscHdfsCreateInsertDir(fs, uri->path);
+ if (FscHdfsHasErrorRaised(fs))
+ {
+ CatchedError *ce = FscHdfsGetFileSystemError(fs);
+ elog(ERROR, "hdfsprotocol_validate : "
+ "failed to create directory %s : %s(%d)",
+ uri->path,
+ ce->errMessage, ce->errCode);
+ }
+ }
- /* destroy fs instance */
- hdfsDisconnect(fs);
+ /* destroy fs instance */
+ FscHdfsFreeFileSystemC(&fs);
}
/* Clean up temporarily created instances */
- FreeExternalTableUri(uri);
+ pfree(uri);
if (nnaddr != NULL)
{
pfree(nnaddr);
diff --git a/src/backend/catalog/gp_toolkit.sql.in b/src/backend/catalog/gp_toolkit.sql.in
index a620284..da3b75f 100644
--- a/src/backend/catalog/gp_toolkit.sql.in
+++ b/src/backend/catalog/gp_toolkit.sql.in
@@ -179,7 +179,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX_AUX%%user_data_tables_readable TO public;
-- External table that determines the local segment id
--
--------------------------------------------------------------------------------
--- CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%localid
+-- CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%localid
-- (
-- localid int
-- )
@@ -196,7 +196,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX_AUX%%user_data_tables_readable TO public;
-- External table that determines the master's segment id
--
--------------------------------------------------------------------------------
--- CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%masterid
+-- CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%masterid
-- (
-- masterid int
-- )
@@ -238,7 +238,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX_AUX%%user_data_tables_readable TO public;
-- External table to read segment log; requires superuser privilege
--
--------------------------------------------------------------------------------
--- CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_segment_ext
+-- CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_segment_ext
-- (
-- logtime timestamp with time zone,
-- loguser text,
@@ -285,7 +285,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX_AUX%%user_data_tables_readable TO public;
-- External table to read the master log; requires superuser privilege
--
--------------------------------------------------------------------------------
-CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_master_ext
+CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_master_ext
(
logtime timestamp with time zone,
loguser text,
@@ -319,7 +319,7 @@ CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX_AUX%%log_master_ext
logstack text
)
EXECUTE E'cat $GP_SEG_DATADIR/pg_log/*.csv' ON MASTER
-FORMAT 'CSV' (DELIMITER AS ',' NULL AS '' QUOTE AS '"');
+FORMAT 'CSV' (DELIMITER ',' NULL '' QUOTE '"');
REVOKE ALL ON TABLE %%JETPACK_PREFIX_AUX%%log_master_ext FROM public;
@@ -994,7 +994,7 @@ GRANT SELECT ON TABLE %%JETPACK_PREFIX%%stats_missing TO public;
-- External table to determine free space on disk on a per-segment basis
--
--------------------------------------------------------------------------------
--- CREATE EXTERNAL WEB TABLE %%JETPACK_PREFIX%%disk_free
+-- CREATE READABLE EXTERNAL WEB TABLE %%JETPACK_PREFIX%%disk_free
-- (
-- dfsegment int,
-- dfhostname text,
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index b443bca..4dbfcb2 100755
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -3360,6 +3360,70 @@ CreateStmt: CREATE OptTemp TABLE qualified_name '(' OptTableElementList ')'
$$ = (Node *)n;
}
+ | CREATE OptTemp TABLE qualified_name '(' OptTableElementList ')'
+ ExtTypedesc FORMAT Sconst format_opt ext_opt_encoding_list OptDistributedBy OptTabPartitionBy
+ {
+ CreateExternalStmt *n = makeNode(CreateExternalStmt);
+ n->iswritable = TRUE;
+ n->isexternal = FALSE;
+ n->isweb = FALSE;
+ $4->istemp = $2;
+ n->base.relation = $4;
+ n->base.tableElts = $6;
+ n->exttypedesc = $8;
+ n->format = $10;
+ n->base.options = $11;
+ n->encoding = $12;
+ n->sreh = NULL;
+ n->base.distributedBy = $13;
+ n->base.partitionBy = $14;
+ n->policy = 0;
+
+ /* various syntax checks for internal table implemented using pluggable storage */
+ if(((ExtTableTypeDesc *) n->exttypedesc)->exttabletype != EXTTBL_TYPE_UNKNOWN)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("LOCATION or EXECUTE may not be used for native table"),
+ errhint("Use CREATE TABLE FORMAT or CREATE EXTERNAL TABLE instead"),
+ errOmitLocation(true)));
+ }
+
+ ListCell *cell;
+ foreach (cell, n->base.tableElts)
+ {
+ Node *node = (Node *)lfirst(cell);
+ switch (nodeTag(node))
+ {
+ case T_ColumnDef:
+ {
+ if (((ColumnDef *)node)->constraints != NIL)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("Do not support column constraint")));
+ }
+ }
+ break;
+
+ case T_Constraint:
+ {
+ if (((Constraint *)node)->contype != CONSTR_PRIMARY)
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("Do not support table constraint other than primary key")));
+ }
+ }
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ $$ = (Node *)n;
+ }
;
/*
@@ -4643,7 +4707,7 @@ CreateExternalStmt: CREATE OptWritable EXTERNAL OptWeb OptTemp TABLE qualified_n
OptWritable: WRITABLE { $$ = TRUE; }
| READABLE { $$ = FALSE; }
- | /*EMPTY*/ { $$ = FALSE; }
+ | /*EMPTY*/ { $$ = TRUE; }
;
OptWeb: WEB { $$ = TRUE; }
@@ -4671,6 +4735,16 @@ ExtTypedesc:
$$ = (Node *)n;
}
+ | /*EMPTY*/
+ {
+ ExtTableTypeDesc *n = makeNode(ExtTableTypeDesc);
+ n->exttabletype = EXTTBL_TYPE_UNKNOWN;
+ n->location_list = NIL;
+ n->command_string = NULL;
+ n->on_clause = NIL;
+
+ $$ = (Node *)n;
+ }
;
ext_on_clause_list: