You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by re...@apache.org on 2020/03/24 12:25:13 UTC
[hbase] branch branch-2.3 updated: HBASE-24032 [RSGroup] Assign
created tables to respective rsgroup automatically instead of manual
operations
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 733a1c0 HBASE-24032 [RSGroup] Assign created tables to respective rsgroup automatically instead of manual operations
733a1c0 is described below
commit 733a1c06a652703e6796a6043447a0c214df81f7
Author: Reid Chan <re...@apache.org>
AuthorDate: Tue Mar 24 20:12:00 2020 +0800
HBASE-24032 [RSGroup] Assign created tables to respective rsgroup automatically instead of manual operations
---
.../hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java | 55 +++++++++
.../hbase/rsgroup/TestRSGroupMappingScript.java | 127 +++++++++++++++++++++
2 files changed, 182 insertions(+)
diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
index 727e860..b869e81 100644
--- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
+++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
@@ -30,6 +30,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HConstants;
@@ -83,6 +84,7 @@ import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.security.access.AccessChecker;
import org.apache.hadoop.hbase.security.access.Permission.Action;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -105,6 +107,48 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
/** Provider for mapping principal names to Users */
private UserProvider userProvider;
+ /** Get rsgroup table mapping script */
+ private RSGroupMappingScript script;
+
+ // Package visibility for testing
+ static class RSGroupMappingScript {
+
+ static final String RS_GROUP_MAPPING_SCRIPT = "hbase.rsgroup.table.mapping.script";
+ static final String RS_GROUP_MAPPING_SCRIPT_TIMEOUT =
+ "hbase.rsgroup.table.mapping.script.timeout";
+
+ private ShellCommandExecutor rsgroupMappingScript;
+
+ RSGroupMappingScript(Configuration conf) {
+ String script = conf.get(RS_GROUP_MAPPING_SCRIPT);
+ if (script == null || script.isEmpty()) {
+ return;
+ }
+
+ rsgroupMappingScript = new ShellCommandExecutor(
+ new String[] { script, "", "" }, null, null,
+ conf.getLong(RS_GROUP_MAPPING_SCRIPT_TIMEOUT, 5000) // 5 seconds
+ );
+ }
+
+ String getRSGroup(String namespace, String tablename) {
+ if (rsgroupMappingScript == null) {
+ return RSGroupInfo.DEFAULT_GROUP;
+ }
+ String[] exec = rsgroupMappingScript.getExecString();
+ exec[1] = namespace;
+ exec[2] = tablename;
+ try {
+ rsgroupMappingScript.execute();
+ } catch (IOException e) {
+ LOG.error(e.getMessage() + " placing back to default rsgroup");
+ return RSGroupInfo.DEFAULT_GROUP;
+ }
+ return rsgroupMappingScript.getOutput().trim();
+ }
+
+ }
+
@Override
public void start(CoprocessorEnvironment env) throws IOException {
if (!(env instanceof HasMasterServices)) {
@@ -123,6 +167,7 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
// set the user-provider.
this.userProvider = UserProvider.instantiate(env.getConfiguration());
+ this.script = new RSGroupMappingScript(env.getConfiguration());
}
@Override
@@ -434,6 +479,16 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
if (groupName == null) {
groupName = RSGroupInfo.DEFAULT_GROUP;
}
+
+ if (groupName.equals(RSGroupInfo.DEFAULT_GROUP)) {
+ TableName tableName = desc.getTableName();
+ groupName = script.getRSGroup(
+ tableName.getNamespaceAsString(),
+ tableName.getQualifierAsString()
+ );
+ LOG.info("rsgroup for " + tableName + " is " + groupName);
+ }
+
RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName);
if (rsGroupInfo == null) {
throw new ConstraintException("Default RSGroup (" + groupName + ") for this table's "
diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMappingScript.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMappingScript.java
new file mode 100644
index 0000000..df2f89b
--- /dev/null
+++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMappingScript.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rsgroup;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint.RSGroupMappingScript;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Category({ SmallTests.class })
+public class TestRSGroupMappingScript {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestRSGroupMappingScript.class);
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestRSGroupMappingScript.class);
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+ private File script;
+
+ @BeforeClass
+ public static void setupScript() throws Exception {
+ String currentDir = new File("").getAbsolutePath();
+ UTIL.getConfiguration().set(
+ RSGroupMappingScript.RS_GROUP_MAPPING_SCRIPT,
+ currentDir + "/rsgroup_table_mapping.sh"
+ );
+ }
+
+ @Before
+ public void setup() throws Exception {
+ script = new File(UTIL.getConfiguration().get(RSGroupMappingScript.RS_GROUP_MAPPING_SCRIPT));
+ if (!script.createNewFile()) {
+ throw new IOException("Can't create script");
+ }
+
+ PrintWriter pw = new PrintWriter(new FileOutputStream(script));
+ try {
+ pw.println("#!/bin/bash");
+ pw.println("namespace=$1");
+ pw.println("tablename=$2");
+ pw.println("if [[ $namespace == test ]]; then");
+ pw.println(" echo test");
+ pw.println("elif [[ $tablename == *foo* ]]; then");
+ pw.println(" echo other");
+ pw.println("else");
+ pw.println(" echo default");
+ pw.println("fi");
+ pw.flush();
+ } finally {
+ pw.close();
+ }
+ boolean executable = script.setExecutable(true);
+ LOG.info("Created " + script + ", executable=" + executable);
+ verifyScriptContent(script);
+ }
+
+ private void verifyScriptContent(File file) throws Exception {
+ BufferedReader reader = new BufferedReader(new FileReader(file));
+ String line;
+ while ((line = reader.readLine()) != null) {
+ LOG.info(line);
+ }
+ }
+
+ @Test
+ public void testScript() throws Exception {
+ RSGroupMappingScript script = new RSGroupMappingScript(UTIL.getConfiguration());
+ TableName testNamespace =
+ TableName.valueOf("test", "should_be_in_test");
+ String rsgroup = script.getRSGroup(
+ testNamespace.getNamespaceAsString(), testNamespace.getQualifierAsString()
+ );
+ Assert.assertEquals("test", rsgroup);
+
+ TableName otherName =
+ TableName.valueOf("whatever", "oh_foo_should_be_in_other");
+ rsgroup = script.getRSGroup(otherName.getNamespaceAsString(), otherName.getQualifierAsString());
+ Assert.assertEquals("other", rsgroup);
+
+ TableName defaultName =
+ TableName.valueOf("nono", "should_be_in_default");
+ rsgroup = script.getRSGroup(
+ defaultName.getNamespaceAsString(), defaultName.getQualifierAsString()
+ );
+ Assert.assertEquals("default", rsgroup);
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (script.exists()) {
+ script.delete();
+ }
+ }
+
+}
+