You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2016/04/25 19:22:26 UTC
ambari git commit: AMBARI-16084. [RU] Hbase is losing table access
permissions during upgrade. (dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/trunk b70758413 -> 51cd067b0
AMBARI-16084. [RU] Hbase is losing table access permissions during upgrade. (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/51cd067b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/51cd067b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/51cd067b
Branch: refs/heads/trunk
Commit: 51cd067b056adb1013a168b59b8d65d1e2946a0c
Parents: b707584
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Mon Apr 25 20:03:09 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Mon Apr 25 20:23:21 2016 +0300
----------------------------------------------------------------------
.../server/upgrade/AbstractUpgradeCatalog.java | 22 ++--
.../server/upgrade/UpgradeCatalog220.java | 4 +-
.../server/upgrade/UpgradeCatalog222.java | 54 ++++++++-
.../server/upgrade/UpgradeCatalog222Test.java | 112 +++++++++++++++++++
4 files changed, 180 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/51cd067b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
index ecd8e3d..2e857ed 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
@@ -90,11 +90,13 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
private static final String CONFIGURATION_TYPE_HIVE_SITE = "hive-site";
private static final String CONFIGURATION_TYPE_HDFS_SITE = "hdfs-site";
- private static final String CONFIGURATION_TYPE_RANGER_KNOX_PLUGIN_PROPERTIES = "ranger-knox-plugin-properties";
+ public static final String CONFIGURATION_TYPE_RANGER_HBASE_PLUGIN_PROPERTIES = "ranger-hbase-plugin-properties";
+ public static final String CONFIGURATION_TYPE_RANGER_KNOX_PLUGIN_PROPERTIES = "ranger-knox-plugin-properties";
private static final String PROPERTY_DFS_NAMESERVICES = "dfs.nameservices";
private static final String PROPERTY_HIVE_SERVER2_AUTHENTICATION = "hive.server2.authentication";
- private static final String PROPERTY_RANGER_KNOX_PLUGIN_ENABLED = "ranger-knox-plugin-enabled";
+ public static final String PROPERTY_RANGER_HBASE_PLUGIN_ENABLED = "ranger-hbase-plugin-enabled";
+ public static final String PROPERTY_RANGER_KNOX_PLUGIN_ENABLED = "ranger-knox-plugin-enabled";
private static final Logger LOG = LoggerFactory.getLogger
(AbstractUpgradeCatalog.class);
@@ -191,18 +193,18 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
return doc;
}
- protected static boolean isRangerKnoxPluginEnabled(Cluster cluster) {
- boolean isRangerKnoxPluginEnabled = false;
+ protected static boolean isConfigEnabled(Cluster cluster, String configType, String propertyName) {
+ boolean isRangerPluginEnabled = false;
if (cluster != null) {
- Config rangerKnoxPluginProperties = cluster.getDesiredConfigByType(CONFIGURATION_TYPE_RANGER_KNOX_PLUGIN_PROPERTIES);
- if (rangerKnoxPluginProperties != null) {
- String rangerKnoxPluginEnabled = rangerKnoxPluginProperties.getProperties().get(PROPERTY_RANGER_KNOX_PLUGIN_ENABLED);
- if (StringUtils.isNotEmpty(rangerKnoxPluginEnabled)) {
- isRangerKnoxPluginEnabled = "yes".equalsIgnoreCase(rangerKnoxPluginEnabled);
+ Config rangerPluginProperties = cluster.getDesiredConfigByType(configType);
+ if (rangerPluginProperties != null) {
+ String rangerPluginEnabled = rangerPluginProperties.getProperties().get(propertyName);
+ if (StringUtils.isNotEmpty(rangerPluginEnabled)) {
+ isRangerPluginEnabled = "yes".equalsIgnoreCase(rangerPluginEnabled);
}
}
}
- return isRangerKnoxPluginEnabled;
+ return isRangerPluginEnabled;
}
protected static class VersionComparator implements Comparator<UpgradeCatalog> {
http://git-wip-us.apache.org/repos/asf/ambari/blob/51cd067b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog220.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog220.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog220.java
index 0eef4f1..2580b0e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog220.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog220.java
@@ -383,7 +383,9 @@ public class UpgradeCatalog220 extends AbstractUpgradeCatalog {
if (!authorizationProviderExists) {
NodeList nodeList = root.getElementsByTagName("gateway");
if (nodeList != null && nodeList.getLength() > 0) {
- boolean rangerPluginEnabled = isRangerKnoxPluginEnabled(cluster);
+ boolean rangerPluginEnabled = isConfigEnabled(cluster,
+ AbstractUpgradeCatalog.CONFIGURATION_TYPE_RANGER_KNOX_PLUGIN_PROPERTIES,
+ AbstractUpgradeCatalog.PROPERTY_RANGER_KNOX_PLUGIN_ENABLED);
Node gatewayNode = nodeList.item(0);
Element newProvider = topologyXml.createElement("provider");
http://git-wip-us.apache.org/repos/asf/ambari/blob/51cd067b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
index 9657d67..1401688 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java
@@ -60,7 +60,6 @@ import com.google.gson.JsonParser;
import com.google.inject.Inject;
import com.google.inject.Injector;
-
/**
* Upgrade catalog for version 2.2.2.
*/
@@ -125,6 +124,10 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
"\\u003dNameNode,name\\" +
"\\u003dFSNamesystem.CorruptBlocks}";
+ public final static String HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES = "hbase.coprocessor.master.classes";
+ public final static String HBASE_SITE_HBASE_COPROCESSOR_REGION_CLASSES = "hbase.coprocessor.region.classes";
+ public final static String HBASE_SITE_HBASE_COPROCESSOR_REGIONSERVER_CLASSES = "hbase.coprocessor.regionserver.classes";
+
// ----- Constructors ------------------------------------------------------
@@ -189,6 +192,7 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
updateHbaseEnvConfig();
updateCorruptedReplicaWidget();
updateZookeeperConfigs();
+ updateHBASEConfigs();
createNewSliderConfigVersion();
initializeStromAndKafkaWidgets();
}
@@ -231,6 +235,54 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
}
}
+ protected void updateHBASEConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Map<String, Cluster> clusterMap = getCheckedClusterMap(ambariManagementController.getClusters());
+
+ for (final Cluster cluster : clusterMap.values()) {
+ Config hbaseSite = cluster.getDesiredConfigByType("hbase-site");
+ boolean rangerHbasePluginEnabled = isConfigEnabled(cluster,
+ AbstractUpgradeCatalog.CONFIGURATION_TYPE_RANGER_HBASE_PLUGIN_PROPERTIES,
+ AbstractUpgradeCatalog.PROPERTY_RANGER_HBASE_PLUGIN_ENABLED);
+ if (hbaseSite != null && rangerHbasePluginEnabled) {
+ Map<String, String> updates = new HashMap<>();
+ String stackVersion = cluster.getCurrentStackVersion().getStackVersion();
+ if (VersionUtils.compareVersions(stackVersion, "2.2") == 0) {
+ if (hbaseSite.getProperties().containsKey(HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES)) {
+ updates.put(HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES,
+ "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor");
+ }
+ if (hbaseSite.getProperties().containsKey(HBASE_SITE_HBASE_COPROCESSOR_REGION_CLASSES)) {
+ updates.put(HBASE_SITE_HBASE_COPROCESSOR_REGION_CLASSES,
+ "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor");
+ }
+ if (hbaseSite.getProperties().containsKey(HBASE_SITE_HBASE_COPROCESSOR_REGIONSERVER_CLASSES)) {
+ updates.put(HBASE_SITE_HBASE_COPROCESSOR_REGIONSERVER_CLASSES,
+ "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint," +
+ "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor");
+ }
+ } else if (VersionUtils.compareVersions(stackVersion, "2.3") == 0) {
+ if (hbaseSite.getProperties().containsKey(HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES)) {
+ updates.put(HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES,
+ "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor ");
+ }
+ if (hbaseSite.getProperties().containsKey(HBASE_SITE_HBASE_COPROCESSOR_REGION_CLASSES)) {
+ updates.put(HBASE_SITE_HBASE_COPROCESSOR_REGION_CLASSES,
+ "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor");
+ }
+ if (hbaseSite.getProperties().containsKey(HBASE_SITE_HBASE_COPROCESSOR_REGIONSERVER_CLASSES)) {
+ updates.put(HBASE_SITE_HBASE_COPROCESSOR_REGIONSERVER_CLASSES,
+ "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint," +
+ "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor");
+ }
+ }
+ if (! updates.isEmpty()) {
+ updateConfigurationPropertiesForCluster(cluster, "hbase-site", updates, true, false);
+ }
+ }
+ }
+ }
+
protected void updateStormConfigs() throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
Map<String, Cluster> clusterMap = getCheckedClusterMap(ambariManagementController.getClusters());
http://git-wip-us.apache.org/repos/asf/ambari/blob/51cd067b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
index 22c7b60..c0c85fd 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java
@@ -138,6 +138,7 @@ public class UpgradeCatalog222Test {
Method updateCorruptedReplicaWidget = UpgradeCatalog222.class.getDeclaredMethod("updateCorruptedReplicaWidget");
Method createNewSliderConfigVersion = UpgradeCatalog222.class.getDeclaredMethod("createNewSliderConfigVersion");
Method updateZookeeperConfigs = UpgradeCatalog222.class.getDeclaredMethod("updateZookeeperConfigs");
+ Method updateHBASEConfigs = UpgradeCatalog222.class.getDeclaredMethod("updateHBASEConfigs");
Method initializeStromAnsKafkaWidgets = UpgradeCatalog222.class.getDeclaredMethod("initializeStromAndKafkaWidgets");
UpgradeCatalog222 upgradeCatalog222 = createMockBuilder(UpgradeCatalog222.class)
@@ -154,6 +155,7 @@ public class UpgradeCatalog222Test {
.addMockedMethod(updateCorruptedReplicaWidget)
.addMockedMethod(createNewSliderConfigVersion)
.addMockedMethod(updateZookeeperConfigs)
+ .addMockedMethod(updateHBASEConfigs)
.addMockedMethod(initializeStromAnsKafkaWidgets)
.createMock();
@@ -169,6 +171,7 @@ public class UpgradeCatalog222Test {
upgradeCatalog222.updateHBASEWidgetDefinition();
upgradeCatalog222.updateCorruptedReplicaWidget();
upgradeCatalog222.updateZookeeperConfigs();
+ upgradeCatalog222.updateHBASEConfigs();
upgradeCatalog222.createNewSliderConfigVersion();
upgradeCatalog222.initializeStromAndKafkaWidgets();
@@ -339,6 +342,115 @@ public class UpgradeCatalog222Test {
easyMockSupport.verifyAll();
}
+ @Test
+ public void testUpdateHBASEConfigs() throws Exception{
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+ final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class);
+ final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class);
+ final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class);
+
+ final Config hbaseSite = easyMockSupport.createNiceMock(Config.class);
+ expect(hbaseSite.getProperties()).andReturn(new HashMap<String, String>(){{
+ put(UpgradeCatalog222.HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES, "test1");
+ put(UpgradeCatalog222.HBASE_SITE_HBASE_COPROCESSOR_REGION_CLASSES, "test2");
+ put(UpgradeCatalog222.HBASE_SITE_HBASE_COPROCESSOR_REGIONSERVER_CLASSES, "test3");
+ }}
+ ).anyTimes();
+
+ final Config rangerHbasePluginProperties = easyMockSupport.createNiceMock(Config.class);
+ expect(rangerHbasePluginProperties.getProperties()).andReturn(new HashMap<String, String>(){{
+ put(AbstractUpgradeCatalog.PROPERTY_RANGER_HBASE_PLUGIN_ENABLED, "yes");
+ }}
+ ).anyTimes();
+
+
+ final Injector mockInjector = Guice.createInjector(new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(AmbariManagementController.class).toInstance(mockAmbariManagementController);
+ bind(Clusters.class).toInstance(mockClusters);
+ bind(EntityManager.class).toInstance(entityManager);
+ bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
+ bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
+ }
+ });
+
+ expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).anyTimes();
+ expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+ put("normal", mockClusterExpected);
+ }}).atLeastOnce();
+
+ UpgradeCatalog222 upgradeCatalog222 = createMockBuilder(UpgradeCatalog222.class)
+ .withConstructor(Injector.class)
+ .withArgs(mockInjector)
+ .addMockedMethod("updateConfigurationPropertiesForCluster", Cluster.class, String.class,
+ Map.class, boolean.class, boolean.class)
+ .createStrictMock();
+
+ // CASE 1 - Ranger enabled, Cluster version is 2.2
+ expect(mockClusterExpected.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.2")).atLeastOnce();
+ expect(mockClusterExpected.getDesiredConfigByType("hbase-site")).andReturn(hbaseSite).atLeastOnce();
+ expect(mockClusterExpected.getDesiredConfigByType(AbstractUpgradeCatalog.CONFIGURATION_TYPE_RANGER_HBASE_PLUGIN_PROPERTIES)).
+ andReturn(rangerHbasePluginProperties).once();
+
+ Map<String, String> expectedUpdates = new HashMap<>();
+ expectedUpdates.put(UpgradeCatalog222.HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES, "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor");
+ expectedUpdates.put(UpgradeCatalog222.HBASE_SITE_HBASE_COPROCESSOR_REGION_CLASSES, "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor");
+ expectedUpdates.put(UpgradeCatalog222.HBASE_SITE_HBASE_COPROCESSOR_REGIONSERVER_CLASSES,
+ "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint," +
+ "com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor");
+
+ upgradeCatalog222.updateConfigurationPropertiesForCluster(mockClusterExpected, "hbase-site", expectedUpdates,
+ true, false);
+ expectLastCall().once();
+
+ easyMockSupport.replayAll();
+ upgradeCatalog222.updateHBASEConfigs();
+ easyMockSupport.verifyAll();
+
+ // CASE 2 - Ranger enabled, Cluster version is 2.3
+ reset(mockClusterExpected, upgradeCatalog222);
+ expect(mockClusterExpected.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.3")).atLeastOnce();
+ expect(mockClusterExpected.getDesiredConfigByType("hbase-site")).andReturn(hbaseSite).atLeastOnce();
+ expect(mockClusterExpected.getDesiredConfigByType(AbstractUpgradeCatalog.CONFIGURATION_TYPE_RANGER_HBASE_PLUGIN_PROPERTIES)).
+ andReturn(rangerHbasePluginProperties).once();
+
+ expectedUpdates = new HashMap<>();
+ expectedUpdates.put(UpgradeCatalog222.HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES, "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor ");
+ expectedUpdates.put(UpgradeCatalog222.HBASE_SITE_HBASE_COPROCESSOR_REGION_CLASSES, "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor");
+ expectedUpdates.put(UpgradeCatalog222.HBASE_SITE_HBASE_COPROCESSOR_REGIONSERVER_CLASSES,
+ "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint," +
+ "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor");
+
+ upgradeCatalog222.updateConfigurationPropertiesForCluster(mockClusterExpected, "hbase-site", expectedUpdates,
+ true, false);
+ expectLastCall().once();
+
+ replay(mockClusterExpected, upgradeCatalog222);
+ upgradeCatalog222.updateHBASEConfigs();
+ easyMockSupport.verifyAll();
+
+ // CASE 3 - Ranger enabled, Cluster version is 2.1
+ reset(mockClusterExpected, upgradeCatalog222);
+ expect(mockClusterExpected.getCurrentStackVersion()).andReturn(new StackId("HDP", "2.1")).atLeastOnce();
+ expect(mockClusterExpected.getDesiredConfigByType("hbase-site")).andReturn(hbaseSite).atLeastOnce();
+ expect(mockClusterExpected.getDesiredConfigByType(AbstractUpgradeCatalog.CONFIGURATION_TYPE_RANGER_HBASE_PLUGIN_PROPERTIES)).
+ andReturn(rangerHbasePluginProperties).once();
+
+ replay(mockClusterExpected, upgradeCatalog222);
+ upgradeCatalog222.updateHBASEConfigs();
+ easyMockSupport.verifyAll();
+
+ // CASE 4 - Ranger disabled
+ reset(mockClusterExpected, upgradeCatalog222);
+ expect(mockClusterExpected.getDesiredConfigByType("hbase-site")).andReturn(hbaseSite).atLeastOnce();
+ expect(mockClusterExpected.getDesiredConfigByType(AbstractUpgradeCatalog.CONFIGURATION_TYPE_RANGER_HBASE_PLUGIN_PROPERTIES)).
+ andReturn(null).once();
+
+ replay(mockClusterExpected, upgradeCatalog222);
+ upgradeCatalog222.updateHBASEConfigs();
+ easyMockSupport.verifyAll();
+ }
@Test
public void testAmsSiteUpdateConfigs() throws Exception{