You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by es...@apache.org on 2017/02/03 09:00:12 UTC
[10/50] [abbrv] incubator-hawq git commit: HAWQ-1256. Enhance libcurl
connection to Ranger Plugin Service,
keep it as a long-live connection in session level
HAWQ-1256. Enhance libcurl connection to Ranger Plugin Service, keep it as a long-live connection in session level
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ad718734
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ad718734
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ad718734
Branch: refs/heads/2.1.0.0-incubating
Commit: ad718734898da2128ad47689243cc08043035573
Parents: 2f5910f
Author: stanlyxiang <st...@gmail.com>
Authored: Tue Jan 10 11:08:01 2017 +0800
Committer: Wen Lin <wl...@pivotal.io>
Committed: Wed Jan 11 15:51:32 2017 +0800
----------------------------------------------------------------------
src/backend/libpq/rangerrest.c | 78 ++++++++++++++++---------------------
src/backend/tcop/postgres.c | 41 +++++++++++++++++++
src/include/utils/rangerrest.h | 7 +++-
3 files changed, 81 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ad718734/src/backend/libpq/rangerrest.c
----------------------------------------------------------------------
diff --git a/src/backend/libpq/rangerrest.c b/src/backend/libpq/rangerrest.c
index 56d30b5..fd8937a 100644
--- a/src/backend/libpq/rangerrest.c
+++ b/src/backend/libpq/rangerrest.c
@@ -71,9 +71,11 @@ static void getClientIP(char *remote_host)
RangerACLResult parse_ranger_response(char* buffer)
{
if (buffer == NULL || strlen(buffer) == 0)
+ {
return RANGERCHECK_UNKNOWN;
+ }
- elog(LOG, "read from Ranger Restful API: %s", buffer);
+ elog(DEBUG3, "parse ranger restful response content : %s", buffer);
struct json_object *response = json_tokener_parse(buffer);
if (response == NULL)
@@ -90,7 +92,7 @@ RangerACLResult parse_ranger_response(char* buffer)
}
int arraylen = json_object_array_length(accessObj);
- elog(LOG, "Array Length: %d",arraylen);
+ elog(DEBUG3, "parse ranger response result array length: %d",arraylen);
// here should return which table's acl check failed in future.
for (int i=0; i< arraylen; i++){
@@ -161,7 +163,7 @@ json_object *create_ranger_request_json(List *args)
AclObjectKind kind = arg_ptr->kind;
char* object = arg_ptr->object;
Assert(user != NULL && object != NULL && privilege != NULL && arg_ptr->isAll);
- elog(LOG, "build json for ranger request, user:%s, kind:%s, object:%s",
+ elog(DEBUG3, "build json for ranger restful request, user:%s, kind:%s, object:%s",
user, AclObjectKindStr[kind], object);
json_object *jelement = json_object_new_object();
@@ -281,29 +283,29 @@ static size_t write_callback(char *contents, size_t size, size_t nitems,
CURL_HANDLE curl = (CURL_HANDLE) userp;
Assert(curl != NULL);
- if (curl->response.buffer == NULL)
+ elog(DEBUG3, "ranger restful response size is %d. response buffer size is %d.", curl->response.response_size, curl->response.buffer_size);
+ int original_size = curl->response.buffer_size;
+ while(curl->response.response_size + realsize >= curl->response.buffer_size)
{
- curl->response.buffer = palloc0(realsize + 1);
+ /*double the buffer size if the buffer is not enough.*/
+ curl->response.buffer_size = curl->response.buffer_size * 2;
}
- else
+ if(original_size < curl->response.buffer_size)
{
- /*Note:*/
- /*our repalloc is not same as realloc, repalloc's first param(buffer) can not be NULL*/
- curl->response.buffer = repalloc(curl->response.buffer, curl->response.size + realsize + 1);
+ /* our repalloc is not same as realloc, repalloc's first param(buffer) can not be NULL */
+ curl->response.buffer = repalloc(curl->response.buffer, curl->response.buffer_size);
}
-
+ elog(DEBUG3, "ranger restful response size is %d. response buffer size is %d.", curl->response.response_size, curl->response.buffer_size);
if (curl->response.buffer == NULL)
{
/* out of memory! */
elog(WARNING, "not enough memory for Ranger response");
return 0;
}
-
- memcpy(curl->response.buffer + curl->response.size, contents, realsize);
- curl->response.size += realsize;
- curl->response.buffer[curl->response.size] = '\0';
- elog(LOG, "read from Ranger Restful API: %s", curl->response.buffer);
-
+ memcpy(curl->response.buffer + curl->response.response_size, contents, realsize);
+ elog(DEBUG3, "read from ranger restful response: %s", curl->response.buffer);
+ curl->response.response_size += realsize;
+ curl->response.buffer[curl->response.response_size] = '\0';
return realsize;
}
@@ -316,15 +318,14 @@ int call_ranger_rest(CURL_HANDLE curl_handle, const char* request)
CURLcode res;
Assert(request != NULL);
- curl_global_init(CURL_GLOBAL_ALL);
-
- /* init the curl session */
- curl_handle->curl_handle = curl_easy_init();
- if (curl_handle->curl_handle == NULL)
- {
- goto _exit;
- }
-
+ /*
+ * Re-initializes all options previously set on a specified CURL handle
+ * to the default values. This puts back the handle to the same state as
+ * it was in when it was just created with curl_easy_init.It does not
+ * change the following information kept in the handle: live connections,
+ * the Session ID cache, the DNS cache, the cookies and shares.
+ */
+ curl_easy_reset(curl_handle->curl_handle);
/* timeout: hard-coded temporarily and maybe should be a guc in future */
curl_easy_setopt(curl_handle->curl_handle, CURLOPT_TIMEOUT, 30L);
@@ -364,19 +365,10 @@ int call_ranger_rest(CURL_HANDLE curl_handle, const char* request)
else
{
ret = 0;
- elog(LOG, "%d bytes retrieved from Ranger Restful API.",
- curl_handle->response.size);
- }
-
-_exit:
- /* cleanup curl stuff */
- if (curl_handle->curl_handle)
- {
- curl_easy_cleanup(curl_handle->curl_handle);
+ elog(DEBUG3, "retrieved %d bytes from ranger restful response.",
+ curl_handle->response.response_size);
}
- /* we're done with libcurl, so clean it up */
- curl_global_cleanup();
return ret;
}
@@ -388,13 +380,11 @@ int check_privilege_from_ranger(List *arg_list)
json_object* jrequest = create_ranger_request_json(arg_list);
Assert(jrequest != NULL);
const char *request = json_object_to_json_string(jrequest);
- elog(LOG, "Send JSON request to Ranger: %s", request);
+ elog(DEBUG3, "send json request to ranger : %s", request);
Assert(request != NULL);
- struct curl_context_t curl_context;
- memset(&curl_context, 0, sizeof(struct curl_context_t));
/* call GET method to send request*/
- if (call_ranger_rest(&curl_context, request) < 0)
+ if (call_ranger_rest(&curl_context_ranger, request) < 0)
{
return RANGERCHECK_NO_PRIV;
}
@@ -403,11 +393,11 @@ int check_privilege_from_ranger(List *arg_list)
json_object_put(jrequest);
/* parse the JSON-format result */
- RangerACLResult ret = parse_ranger_response(curl_context.response.buffer);
- /* free response buffer */
- if (curl_context.response.buffer != NULL)
+ RangerACLResult ret = parse_ranger_response(curl_context_ranger.response.buffer);
+ if (curl_context_ranger.response.buffer != NULL)
{
- pfree(curl_context.response.buffer);
+ /* reset response size to reuse the buffer. */
+ curl_context_ranger.response.response_size = 0;
}
return ret;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ad718734/src/backend/tcop/postgres.c
----------------------------------------------------------------------
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 5a8327e..c8d7e33 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -114,6 +114,8 @@
#include "cdb/cdbinmemheapam.h"
+#include "utils/rangerrest.h"
+
#include "resourcemanager/dynrm.h"
#include "resourcemanager/envswitch.h"
#include "resourcemanager/communication/rmcomm_QD2RM.h"
@@ -133,6 +135,7 @@ extern char *optarg;
extern char *savedSeqServerHost;
extern int savedSeqServerPort;
+struct curl_context_t curl_context_ranger;
/* ----------------
* global variables
* ----------------
@@ -266,6 +269,7 @@ static void log_disconnections(int code, Datum arg);
static bool renice_current_process(int nice_level);
static int getSlaveHostNumber(FILE *fp);
static bool CheckSlaveFile();
+static void curl_finalize(int code, Datum arg);
/*saved interrupt global variable for client_read_xxx functions*/
static bool SavedImmediateInterruptOK = false;
@@ -4626,6 +4630,25 @@ PostgresMain(int argc, char *argv[], const char *username)
if (!ignore_till_sync)
send_ready_for_query = true; /* initially, or after error */
+ /* for enable ranger*/
+ if (AmIMaster() && enable_ranger && !curl_context_ranger.hasInited)
+ {
+ memset(&curl_context_ranger, 0, sizeof(curl_context_t));
+ curl_global_init(CURL_GLOBAL_ALL);
+ /* init the curl session */
+ curl_context_ranger.curl_handle = curl_easy_init();
+ if (curl_context_ranger.curl_handle == NULL) {
+ /* cleanup curl stuff */
+ /* no need to cleanup curl_handle since it's null. just cleanup curl global.*/
+ curl_global_cleanup();
+ elog(ERROR, "initialize global curl context failed.");
+ }
+ curl_context_ranger.hasInited = true;
+ curl_context_ranger.response.buffer = palloc0(CURL_RES_BUFFER_SIZE);
+ curl_context_ranger.response.buffer_size = CURL_RES_BUFFER_SIZE;
+ elog(DEBUG3, "initialize global curl context for privileges check.");
+ on_proc_exit(curl_finalize, 0);
+ }
/*
* Non-error queries loop here.
*/
@@ -5314,6 +5337,24 @@ PostgresMain(int argc, char *argv[], const char *username)
return 1; /* keep compiler quiet */
}
+static void
+curl_finalize(int code, Datum arg __MAYBE_UNUSED)
+{
+ if (AmIMaster() && curl_context_ranger.hasInited)
+ {
+ if (curl_context_ranger.response.buffer != NULL) {
+ pfree(curl_context_ranger.response.buffer);
+ }
+ /* cleanup curl stuff */
+ if (curl_context_ranger.curl_handle) {
+ curl_easy_cleanup(curl_context_ranger.curl_handle);
+ }
+ /* we're done with libcurl, so clean it up */
+ curl_global_cleanup();
+ curl_context_ranger.hasInited = false;
+ elog(DEBUG3, "finalize the global struct for curl handle context.");
+ }
+}
/*
* Obtain platform stack depth limit (in bytes)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ad718734/src/include/utils/rangerrest.h
----------------------------------------------------------------------
diff --git a/src/include/utils/rangerrest.h b/src/include/utils/rangerrest.h
index 692c832..f67d8e5 100644
--- a/src/include/utils/rangerrest.h
+++ b/src/include/utils/rangerrest.h
@@ -37,6 +37,7 @@
#include "tcop/tcopprot.h"
#define HOST_BUFFER_SIZE 1025
+#define CURL_RES_BUFFER_SIZE 1024
typedef enum
{
@@ -59,10 +60,13 @@ typedef struct curl_context_t
struct
{
char* buffer;
- int size;
+ int response_size;
+ int buffer_size;
} response;
char* last_http_reponse;
+
+ bool hasInited;
} curl_context_t;
typedef curl_context_t* CURL_HANDLE;
@@ -94,5 +98,6 @@ RangerACLResult parse_ranger_response(char *);
json_object *create_ranger_request_json(List *);
int call_ranger_rest(CURL_HANDLE curl_handle, const char *request);
extern int check_privilege_from_ranger(List *);
+extern struct curl_context_t curl_context_ranger;
#endif