You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by wl...@apache.org on 2017/03/10 07:45:27 UTC
incubator-hawq git commit: HAWQ-1380. Keep hawq_toolkit schema check
in HAWQ native side
Repository: incubator-hawq
Updated Branches:
refs/heads/master a80e9e8ca -> 6c012e999
HAWQ-1380. Keep hawq_toolkit schema check in HAWQ native side
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6c012e99
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6c012e99
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6c012e99
Branch: refs/heads/master
Commit: 6c012e9994fdc27b6d515ecd63fa1525880f4aa4
Parents: a80e9e8
Author: interma <in...@outlook.com>
Authored: Wed Mar 8 12:41:30 2017 +0800
Committer: Wen Lin <wl...@pivotal.io>
Committed: Fri Mar 10 15:44:22 2017 +0800
----------------------------------------------------------------------
src/backend/catalog/aclchk.c | 21 +++++--
src/backend/utils/misc/guc.c | 8 ++-
src/include/utils/guc.h | 4 +-
.../feature/Ranger/ans/normal10000_success.ans | 10 ++++
src/test/feature/Ranger/policy/10000/1.json | 1 +
src/test/feature/Ranger/sql/manual/10000.sql | 5 ++
src/test/feature/Ranger/test_ranger.cpp | 61 ++++++++++++++++++--
src/test/feature/sanity_tests.txt | 2 +-
8 files changed, 97 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/backend/catalog/aclchk.c
----------------------------------------------------------------------
diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c
index 16e00c1..e3d4d61 100644
--- a/src/backend/catalog/aclchk.c
+++ b/src/backend/catalog/aclchk.c
@@ -2685,8 +2685,12 @@ List *getActionName(AclMode mask)
bool checkNamespaceFallback(Oid x)
{
- if (x == PG_CATALOG_NAMESPACE || x == information_schema_namespcace_oid
- || x == PG_AOSEGMENT_NAMESPACE || x == PG_TOAST_NAMESPACE || x == PG_BITMAPINDEX_NAMESPACE)
+ if (x == PG_CATALOG_NAMESPACE
+ || x == PG_AOSEGMENT_NAMESPACE
+ || x == PG_TOAST_NAMESPACE
+ || x == PG_BITMAPINDEX_NAMESPACE
+ || x == information_schema_namespace_oid
+ || x == hawq_toolkit_schema_namespace_oid )
{
return true;
}
@@ -2707,13 +2711,18 @@ bool checkNamespaceFallback(Oid x)
bool fallBackToNativeCheck(AclObjectKind objkind, Oid obj_oid, Oid roleid, AclMode mode)
{
- /* get the latest information_schema_namespcace_oid. Since caql access heap table
- * directly without aclcheck, this function will not be called recursively
+ /* get the latest information_schema_namespace_oid and hawq_toolkit_schema_namespace_oid.
+ * Since caql access heap table directly without aclcheck, this function will not be called recursively
*/
- if (information_schema_namespcace_oid == 0)
+ if (information_schema_namespace_oid == 0)
{
- information_schema_namespcace_oid = (int)get_namespace_oid("information_schema");
+ information_schema_namespace_oid = (int)get_namespace_oid("information_schema");
}
+ if (hawq_toolkit_schema_namespace_oid == 0)
+ {
+ hawq_toolkit_schema_namespace_oid = (int)get_namespace_oid("hawq_toolkit");
+ }
+
/* for heap table, we fall back to native check. */
if (objkind == ACL_KIND_CLASS)
{
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index bd03d5e..2c2b918 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -768,7 +768,9 @@ bool optimizer_prefer_scalar_dqa_multistage_agg;
bool optimizer_parallel_union;
bool optimizer_array_constraints;
-int information_schema_namespcace_oid;
+/* fallback in ranger ACL check */
+int information_schema_namespace_oid;
+int hawq_toolkit_schema_namespace_oid;
/* Security */
bool gp_reject_internal_tcp_conn = true;
@@ -6188,11 +6190,11 @@ static struct config_int ConfigureNamesInt[] =
},
{
- {"information_schema_namespcace_oid", PGC_USERSET, DEVELOPER_OPTIONS,
+ {"information_schema_namespace_oid", PGC_USERSET, DEVELOPER_OPTIONS,
gettext_noop("the oid of information_schema namespace"),
NULL
},
- &information_schema_namespcace_oid,
+ &information_schema_namespace_oid,
0, 0, INT_MAX, NULL, NULL
},
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/include/utils/guc.h
----------------------------------------------------------------------
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index 77cee1e..86154ee 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -445,7 +445,9 @@ extern bool optimizer_parallel_union;
extern bool optimizer_array_constraints;
-extern int information_schema_namespcace_oid;
+/* fallback in ranger ACL check */
+extern int information_schema_namespace_oid;
+extern int hawq_toolkit_schema_namespace_oid;
/**
* Enable logging of DPE match in optimizer.
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/test/feature/Ranger/ans/normal10000_success.ans
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/ans/normal10000_success.ans b/src/test/feature/Ranger/ans/normal10000_success.ans
new file mode 100644
index 0000000..125e649
--- /dev/null
+++ b/src/test/feature/Ranger/ans/normal10000_success.ans
@@ -0,0 +1,10 @@
+-- start_ignore
+-- end_ignore
+set session role=usertest10000;
+SET
+select count(*) from information_schema.view_table_usage;
+0
+select count(*) from hawq_toolkit.hawq_table_indexes;
+0
+select count(*) from pg_catalog.pg_compression;
+4
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/test/feature/Ranger/policy/10000/1.json
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/policy/10000/1.json b/src/test/feature/Ranger/policy/10000/1.json
new file mode 100644
index 0000000..e3a4d86
--- /dev/null
+++ b/src/test/feature/Ranger/policy/10000/1.json
@@ -0,0 +1 @@
+{"allowExceptions": [], "denyExceptions": [], "denyPolicyItems": [], "description": "no description", "isAuditEnabled": true, "isEnabled": true, "name": "policy10000-1", "policyItems": [{"accesses": [{"isAllowed": true, "type": "usage-schema"}, {"isAllowed": true, "type": "create"}], "conditions": [], "delegateAdmin": true, "groups": null, "users": ["usertest10000", "usersuper10000"]}], "resources": {"database": {"isExcludes": false, "isRecursive": false, "values": ["hawq_feature_test_db"]}, "schema": {"isExcludes": false, "isRecursive": false, "values": ["public"]}, "table": {"isExcludes": false, "isRecursive": false, "values": ["*"]}}, "service": "hawq", "version": 1}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/test/feature/Ranger/sql/manual/10000.sql
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/sql/manual/10000.sql b/src/test/feature/Ranger/sql/manual/10000.sql
new file mode 100644
index 0000000..3912cb3
--- /dev/null
+++ b/src/test/feature/Ranger/sql/manual/10000.sql
@@ -0,0 +1,5 @@
+set session role=usertest10000;
+select count(*) from information_schema.view_table_usage;
+select count(*) from hawq_toolkit.hawq_table_indexes;
+select count(*) from pg_catalog.pg_compression;
+
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/test/feature/Ranger/test_ranger.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/Ranger/test_ranger.cpp b/src/test/feature/Ranger/test_ranger.cpp
index acc8b97..93e8452 100644
--- a/src/test/feature/Ranger/test_ranger.cpp
+++ b/src/test/feature/Ranger/test_ranger.cpp
@@ -41,7 +41,7 @@ TEST_F(TestHawqRanger, BasicTest) {
gpdfist.init_gpfdist();
string rootPath(util.getTestRootPath());
- string initfile = hawq::test::stringFormat("Ranger/sql/init_file");
+ string initfile = "Ranger/sql/init_file";
auto cmd = hawq::test::stringFormat("ls -l %s/Ranger/sql/normal/*.sql 2>/dev/null | grep \"^-\" | wc -l", rootPath.c_str());
int sql_num = std::atoi(Command::getCommandOutput(cmd).c_str());
int writableTableCase = 28;
@@ -121,9 +121,6 @@ TEST_F(TestHawqRanger, BasicTest) {
string normal_ansfile_success = hawq::test::stringFormat("Ranger/ans/normal%d_success.ans", i);
string super_ansfile_success = hawq::test::stringFormat("Ranger/ans/super%d_success.ans", i);
-
- cmd = hawq::test::stringFormat("ls -l %s/Ranger/policy/%d/ 2>/dev/null| grep \"^-\" | wc -l", rootPath.c_str(), i);
- int policy_num = std::atoi(Command::getCommandOutput(cmd).c_str());
cmd = hawq::test::stringFormat("ls -l %s/Ranger/sql/super/%d.sql 2>/dev/null | grep \"^-\" | wc -l", rootPath.c_str(), i);
int supersqlexist = std::atoi(Command::getCommandOutput(cmd).c_str());
util.execSQLFile(normal_sqlfile, normal_ansfile_success, initfile, true, true);
@@ -157,3 +154,59 @@ TEST_F(TestHawqRanger, BasicTest) {
gpdfist.finalize_gpfdist();
}
}
+
+static void clear_env(SQLUtility &util, int sql_id, string rootPath, string rangerHost)
+{
+ int i = sql_id;
+ // delete user_num
+ std::string normalusername = hawq::test::stringFormat("usertest%d", i);
+ std::string superusername = hawq::test::stringFormat("usersuper%d", i);
+ util.execute(hawq::test::stringFormat("drop role %s;",normalusername.c_str()), false);
+ util.execute(hawq::test::stringFormat("drop role %s;",superusername.c_str()), false);
+
+ // delete policy
+ std::string cmd = hawq::test::stringFormat("ls -l %s/Ranger/policy/%d/ 2>/dev/null| grep \"^-\" | wc -l ", rootPath.c_str(), i);
+ int policy_num = std::atoi(Command::getCommandOutput(cmd).c_str());
+ for (int j = 1; j <= policy_num; j++) {
+ cmd = hawq::test::stringFormat("python %s/Ranger/rangerpolicy.py -h %s -d policy%d-%d", rootPath.c_str(), rangerHost.c_str(), i, j);
+ Command::getCommandStatus(cmd);
+ }
+}
+
+TEST_F(TestHawqRanger, FallbackTest) {
+ SQLUtility util;
+
+ if (util.getGUCValue("hawq_acl_type") == "ranger")
+ {
+ string rootPath(util.getTestRootPath());
+ string rangerHost = RANGER_HOST;
+ string initfile = "Ranger/sql/init_file";
+ string cmd;
+ int FallbackCase = 10000;
+ int i = FallbackCase;
+
+ // clear environment
+ clear_env(util, i, rootPath, rangerHost);
+
+ // create user_num
+ std::string normalusername = hawq::test::stringFormat("usertest%d", i);;
+ std::string superusername = hawq::test::stringFormat("usersuper%d", i);;
+ util.execute(hawq::test::stringFormat("create role %s with login createdb;", normalusername.c_str()),true);
+ util.execute(hawq::test::stringFormat("create role %s with login createdb superuser;", superusername.c_str()),true);
+ // add user
+ cmd = hawq::test::stringFormat("python %s/Ranger/rangeruser.py -h %s -u %s,%s", rootPath.c_str(),
+ rangerHost.c_str(), normalusername.c_str(), superusername.c_str());
+ Command::getCommandStatus(cmd);
+ // add policy
+ cmd = hawq::test::stringFormat("python %s/Ranger/rangerpolicy.py -h %s -a %s/Ranger/policy/%d/%d.json", rootPath.c_str(), rangerHost.c_str(), rootPath.c_str(), i, 1);
+ Command::getCommandStatus(cmd);
+ sleep(60);
+
+ // run sql test
+ string normal_sqlfile = hawq::test::stringFormat("Ranger/sql/manual/%d.sql", i);
+ string normal_ansfile_success = hawq::test::stringFormat("Ranger/ans/normal%d_success.ans", i);
+ util.execSQLFile(normal_sqlfile, normal_ansfile_success, initfile, true, true);
+
+ clear_env(util, i, rootPath, rangerHost);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6c012e99/src/test/feature/sanity_tests.txt
----------------------------------------------------------------------
diff --git a/src/test/feature/sanity_tests.txt b/src/test/feature/sanity_tests.txt
index 1c58281..bc6df0b 100644
--- a/src/test/feature/sanity_tests.txt
+++ b/src/test/feature/sanity_tests.txt
@@ -3,4 +3,4 @@
#you can have several PARALLEL or SRRIAL
PARALLEL=TestErrorTable.*:TestPreparedStatement.*:TestUDF.*:TestAOSnappy.*:TestAlterOwner.*:TestAlterTable.*:TestCreateTable.*:TestGuc.*:TestType.*:TestDatabase.*:TestParquet.*:TestPartition.*:TestSubplan.*:TestAggregate.*:TestCreateTypeComposite.*:TestGpDistRandom.*:TestInformationSchema.*:TestQueryInsert.*:TestQueryNestedCaseNull.*:TestQueryPolymorphism.*:TestQueryPortal.*:TestQueryPrepare.*:TestQuerySequence.*:TestCommonLib.*:TestToast.*:TestTransaction.*:TestCommand.*:TestCopy.*:TestHawqRegister.TestPartitionTableMultilevel:TestHawqRegister.TestUsage1ExpectSuccessDifferentSchema:TestHawqRegister.TestUsage1ExpectSuccess:TestHawqRegister.TestUsage1SingleHawqFile:TestHawqRegister.TestUsage1SingleHiveFile:TestHawqRegister.TestDataTypes:TestHawqRegister.TestUsage1EofSuccess:TestHawqRegister.TestUsage2Case1Expected:TestHawqRegister.TestUsage2Case2Expected
-SERIAL=TestHawqRanger.BasicTest:TestExternalOid.TestExternalOidAll:TestExternalTable.TestExternalTableAll:TestTemp.BasicTest:TestRowTypes.*
+SERIAL=TestHawqRanger.*:TestExternalOid.TestExternalOidAll:TestExternalTable.TestExternalTableAll:TestTemp.BasicTest:TestRowTypes.*