You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by od...@apache.org on 2016/03/31 02:49:52 UTC
incubator-hawq git commit: HAWQ-546. Implemented call of
pxf_get_object_fields for Hive on psql.
Repository: incubator-hawq
Updated Branches:
refs/heads/master 7d155bee0 -> f86ead491
HAWQ-546. Implemented call of pxf_get_object_fields for Hive on psql.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f86ead49
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f86ead49
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f86ead49
Branch: refs/heads/master
Commit: f86ead49195513e4876688e135c05d17727c7587
Parents: 7d155be
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Tue Mar 29 14:44:05 2016 -0700
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Wed Mar 30 17:45:43 2016 -0700
----------------------------------------------------------------------
src/bin/psql/describe.c | 143 ++++++++++++++++++++
src/test/regress/input/hcatalog_lookup.source | 4 +
src/test/regress/output/hcatalog_lookup.source | 7 +
3 files changed, 154 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f86ead49/src/bin/psql/describe.c
----------------------------------------------------------------------
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index 99718d2..4ba1e2d 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -42,11 +42,16 @@ static bool describeOneTSConfig(const char *oid, const char *nspname,
static void printACLColumn(PQExpBuffer buf, const char *colname);
static bool isGPDB(void);
static bool isGPDB4200OrLater(void);
+static bool describePxfTable(const char *profile, const char *pattern, bool verbose);
+static void parsePxfPattern(const char *user_pattern, char **pattern);
/* GPDB 3.2 used PG version 8.2.10, and we've moved the minor number up since then for each release, 4.1 = 8.2.15 */
/* Allow for a couple of future releases. If the version isn't in this range, we are talking to PostgreSQL, not GPDB */
#define mightBeGPDB() (pset.sversion >= 80210 && pset.sversion < 80222)
+#define HiveProfileName "Hive"
+#define HcatalogSourceName "hcatalog"
+
static bool isGPDB(void)
{
static enum { gpdb_maybe, gpdb_yes, gpdb_no } talking_to_gpdb;
@@ -1152,6 +1157,24 @@ describeTableDetails(const char *pattern, bool verbose, bool showSystem)
PGresult *res;
int i;
+ //Hive hook in this method
+ if(pattern && strncmp(pattern, HcatalogSourceName, strlen(HcatalogSourceName)) == 0)
+ {
+ char *pxf_pattern = NULL;
+ char *pattern_dup = strdup(pattern);
+ parsePxfPattern(pattern_dup, &pxf_pattern);
+ if (!pxf_pattern)
+ {
+ fprintf(stderr, _("Invalid pattern provided.\n"));
+ free(pattern_dup);
+ return false;
+ }
+
+ bool success = describePxfTable(HiveProfileName, pxf_pattern, verbose);
+ free(pattern_dup);
+ return success;
+ }
+
initPQExpBuffer(&buf);
printfPQExpBuffer(&buf,
@@ -4213,3 +4236,123 @@ printACLColumn(PQExpBuffer buf, const char *colname)
"pg_catalog.array_to_string(%s, '\\n') AS \"%s\"",
colname, gettext_noop("Access privileges"));
}
+
+/*
+ * parsePxfPattern
+ *
+ * Splits user_pattern by "." and writes second part to pattern.
+ */
+static void
+parsePxfPattern(const char *user_pattern, char **pattern)
+{
+ strtok(user_pattern, ".");
+ *pattern = strtok(NULL, "/0");
+}
+
+/*
+ * describePxfTable
+ *
+ * Describes external PXF table.
+ */
+static bool
+describePxfTable(const char *profile, const char *pattern, bool verbose)
+{
+ PQExpBufferData buf;
+ PQExpBufferData title;
+ PGresult *res;
+ printQueryOpt myopt = pset.popt;
+ printTableContent cont;
+ int cols = 0;
+ int total_numrows = 0;
+ char *headers[2];
+ bool printTableInitialized = false;
+
+ char *previous_path = NULL;
+ char *previous_itemname = NULL;
+
+ char *path;
+ char *itemname;
+ char *fieldname;
+ char *fieldtype;
+ int total_fields = 0; //needed to know how much memory allocate for current table
+
+ initPQExpBuffer(&buf);
+
+ printfPQExpBuffer(&buf, "SELECT t.*, COUNT() OVER(PARTITION BY path, itemname) as total_fields FROM\n"
+ "pxf_get_item_fields('%s', '%s') t\n", profile, pattern);
+
+ res = PSQLexec(buf.data, false);
+ total_numrows = PQntuples(res);
+ termPQExpBuffer(&buf);
+ if (!res)
+ return false;
+
+ myopt.nullPrint = NULL;
+ myopt.title = _("List of Hive tables");
+ myopt.translate_header = true;
+
+ /* Header */
+ headers[0] = gettext_noop("Column");
+ headers[1] = gettext_noop("Type");
+ cols = 2;
+
+ for (int i = 0; i < total_numrows; i++)
+ {
+
+ path = PQgetvalue(res, i, 0);
+ itemname = PQgetvalue(res, i, 1);
+ fieldname = PQgetvalue(res, i, 2);
+ fieldtype = PQgetvalue(res, i, 3);
+ total_fields = PQgetvalue(res, i, 4);
+
+ /* First row for current table */
+ if (previous_itemname == NULL
+ || strncmp(previous_itemname, itemname,
+ strlen(previous_itemname)) != 0
+ || strncmp(previous_path, path,
+ strlen(previous_path)) != 0)
+ {
+
+ if (previous_itemname != NULL)
+ printTable(&cont, pset.queryFout, pset.logfile);
+
+ /* Do clean-up for previous tables if any */
+ if (printTableInitialized)
+ {
+ printTableCleanup(&cont);
+ termPQExpBuffer(&title);
+ printTableInitialized = false;
+ }
+
+ /* Initialize */
+ initPQExpBuffer(&title);
+ printfPQExpBuffer(&title, _("PXF %s Table \"%s.%s\""), profile, path, itemname);
+ printTableInit(&cont, &myopt, title.data, cols, total_fields);
+ printTableInitialized = true;
+
+ for (int j = 0; j < cols; j++)
+ printTableAddHeader(&cont, headers[j], true, 'l');
+ }
+
+ /* Column */
+ printTableAddCell(&cont, fieldname, false, false);
+
+ /* Type */
+ printTableAddCell(&cont, fieldtype, false, false);
+
+ previous_path = path;
+ previous_itemname = itemname;
+
+ }
+
+ printTable(&cont, pset.queryFout, pset.logfile);
+
+ if (printTableInitialized)
+ {
+ printTableCleanup(&cont);
+ termPQExpBuffer(&title);
+ }
+
+ PQclear(res);
+ return true;
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f86ead49/src/test/regress/input/hcatalog_lookup.source
----------------------------------------------------------------------
diff --git a/src/test/regress/input/hcatalog_lookup.source b/src/test/regress/input/hcatalog_lookup.source
index 3d7279f..239494a 100644
--- a/src/test/regress/input/hcatalog_lookup.source
+++ b/src/test/regress/input/hcatalog_lookup.source
@@ -7,11 +7,15 @@ SET hcatalog_enable = false;
SELECT * from hcatalog.db.t;
SELECT * FROM pxf_get_item_fields('Hive', '*');
+ \d hcatalog.*.*
-- enable GUC
SET hcatalog_enable = true;
SELECT * FROM pxf_get_item_fields('Hive', '*abc*abc*');
+\d hcatalog.*abc*.*abc*
+\d hcatalog
+\d hcatalog.
-- Create function to insert and scan in-memory data to pg_class
CREATE OR REPLACE FUNCTION convert_to_hcatalog_schema(schemaName text) RETURNS text
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f86ead49/src/test/regress/output/hcatalog_lookup.source
----------------------------------------------------------------------
diff --git a/src/test/regress/output/hcatalog_lookup.source b/src/test/regress/output/hcatalog_lookup.source
index ed4a7ef..533603a 100644
--- a/src/test/regress/output/hcatalog_lookup.source
+++ b/src/test/regress/output/hcatalog_lookup.source
@@ -9,6 +9,8 @@ LINE 1: SELECT * from hcatalog.db.t;
^
SELECT * FROM pxf_get_item_fields('Hive', '*');
ERROR: HCatalog querying is not enabled
+\d hcatalog.*.*
+ERROR: HCatalog querying is not enabled
-- enable GUC
SET hcatalog_enable = true;
SELECT * FROM pxf_get_item_fields('Hive', '*abc*abc*');
@@ -16,6 +18,11 @@ SELECT * FROM pxf_get_item_fields('Hive', '*abc*abc*');
------+----------+-----------+-----------
(0 rows)
+\d hcatalog.*abc*.*abc*
+\d hcatalog
+Invalid pattern provided.
+\d hcatalog.
+Invalid pattern provided.
-- Create function to insert and scan in-memory data to pg_class
CREATE OR REPLACE FUNCTION convert_to_hcatalog_schema(schemaName text) RETURNS text
AS '@abs_builddir@/regress@DLSUFFIX@', 'convert_to_hcatalog_schema'