You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ha...@apache.org on 2017/06/25 09:59:55 UTC
[19/25] ambari git commit: AMBARI-21268 Remove Upgrade Catalogs For
Every Version Before 2.5 (dgrinenko)
http://git-wip-us.apache.org/repos/asf/ambari/blob/af1bf85c/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog2402.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog2402.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog2402.java
deleted file mode 100644
index a1ca8a5..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog2402.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.server.upgrade;
-
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.configuration.Configuration;
-import org.apache.ambari.server.orm.dao.ClusterDAO;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-
-/**
- * Upgrade catalog for version 2.4.0.
- */
-public class UpgradeCatalog2402 extends AbstractUpgradeCatalog {
-
- @Inject
- ClusterDAO clusterDAO;
-
- @Inject
- Configuration config;
-
- /**
- * Logger.
- */
- private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog2402.class);
-
- // ----- Constructors ------------------------------------------------------
-
- /**
- * Don't forget to register new UpgradeCatalogs in {@link SchemaUpgradeHelper.UpgradeHelperModule#configure()}
- *
- * @param injector Guice injector to track dependencies and uses bindings to inject them.
- */
- @Inject
- public UpgradeCatalog2402(Injector injector) {
- super(injector);
- injector.injectMembers(this);
- }
-
- // ----- UpgradeCatalog ----------------------------------------------------
-
- /**
- * {@inheritDoc}
- */
- @Override
- public String getTargetVersion() {
- return "2.4.0.2";
- }
-
- // ----- AbstractUpgradeCatalog --------------------------------------------
-
- /**
- * {@inheritDoc}
- */
- @Override
- public String getSourceVersion() {
- return "2.4.0";
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- protected void executeDDLUpdates() throws AmbariException, SQLException {
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- protected void executePreDMLUpdates() throws AmbariException, SQLException {
-
- }
-
- @Override
- protected void executeDMLUpdates() throws AmbariException, SQLException {
- updateKafkaWidgetDefinition();
- }
-
-
- protected void updateKafkaWidgetDefinition() throws AmbariException {
- LOG.info("Updating Kafka widget definition.");
-
- Map<String, List<String>> widgetMap = new HashMap<>();
- Map<String, String> sectionLayoutMap = new HashMap<>();
-
- List<String> kafkaSummaryWidgets = new ArrayList<>(Arrays.asList("Active Controller Count"));
- widgetMap.put("KAFKA_SUMMARY", kafkaSummaryWidgets);
- sectionLayoutMap.put("KAFKA_SUMMARY", "default_kafka_dashboard");
-
- updateWidgetDefinitionsForService("KAFKA", widgetMap, sectionLayoutMap);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/af1bf85c/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog242.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog242.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog242.java
deleted file mode 100644
index ecf64a2..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog242.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.server.upgrade;
-
-import java.sql.SQLException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.configuration.Configuration;
-import org.apache.ambari.server.orm.DBAccessor;
-import org.apache.ambari.server.orm.dao.PermissionDAO;
-import org.apache.ambari.server.orm.dao.PrincipalDAO;
-import org.apache.ambari.server.orm.dao.PrincipalTypeDAO;
-import org.apache.ambari.server.orm.dao.PrivilegeDAO;
-import org.apache.ambari.server.orm.entities.PermissionEntity;
-import org.apache.ambari.server.orm.entities.PrincipalEntity;
-import org.apache.ambari.server.orm.entities.PrincipalTypeEntity;
-import org.apache.ambari.server.orm.entities.PrivilegeEntity;
-import org.apache.ambari.server.orm.entities.ResourceEntity;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.persist.Transactional;
-
-/**
- * Upgrade catalog for version 2.4.2.
- */
-public class UpgradeCatalog242 extends AbstractUpgradeCatalog {
-
- protected static final String EXTENSION_TABLE = "extension";
- protected static final String USERS_TABLE = "users";
- protected static final String HOST_ROLE_COMMAND_TABLE = "host_role_command";
- protected static final String BLUEPRINT_TABLE = "blueprint";
- protected static final String HOST_GROUP_TABLE = "hostgroup";
- protected static final String BLUEPRINT_CONFIGURATION = "blueprint_configuration";
- protected static final String BLUEPRINT_SETTING = "blueprint_setting";
- protected static final String HOSTGROUP_COMPONENT = "hostgroup_component";
- protected static final String HOSTGROUP_CONFIGURATION = "hostgroup_configuration";
-
- protected static final String BLUEPRINT_NAME_COLUMN = "blueprint_name";
- protected static final String EXTENSION_NAME_COLUMN = "extension_name";
- protected static final String EXTENSION_VERSION_COLUMN = "extension_version";
- protected static final String USER_TYPE_COLUMN = "user_type";
- protected static final String USER_NAME_COLUMN = "user_name";
- protected static final String ROLE_COLUMN = "role";
- protected static final String STATUS_COLUMN = "status";
- protected static final String NAME_COLUMN = "name";
-
-
- /**
- * Logger.
- */
- private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog242.class);
-
-
-
-
- // ----- Constructors ------------------------------------------------------
-
- /**
- * Don't forget to register new UpgradeCatalogs in {@link org.apache.ambari.server.upgrade.SchemaUpgradeHelper.UpgradeHelperModule#configure()}
- *
- * @param injector Guice injector to track dependencies and uses bindings to inject them.
- */
- @Inject
- public UpgradeCatalog242(Injector injector) {
- super(injector);
- this.injector = injector;
- }
-
- // ----- UpgradeCatalog ----------------------------------------------------
-
- /**
- * {@inheritDoc}
- */
- @Override
- public String getTargetVersion() {
- return "2.4.2";
- }
-
- // ----- AbstractUpgradeCatalog --------------------------------------------
-
- /**
- * {@inheritDoc}
- */
- @Override
- public String getSourceVersion() {
- return "2.4.0.2";
- }
-
-
- @Override
- protected void executeDDLUpdates() throws AmbariException, SQLException {
- updateTablesForMysql();
- }
-
- @Override
- protected void executePreDMLUpdates() throws AmbariException, SQLException {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- protected void executeDMLUpdates() throws AmbariException, SQLException {
- addNewConfigurationsFromXml();
- createRoleAuthorizations();
- convertRolePrincipals();
- }
-
- /**
- * Create new role authorizations: CLUSTER.RUN_CUSTOM_COMMAND and AMBARI.RUN_CUSTOM_COMMAND
- *
- * @throws SQLException
- */
- @Transactional
- protected void createRoleAuthorizations() throws SQLException {
- addRoleAuthorization("CLUSTER.RUN_CUSTOM_COMMAND", "Perform custom cluster-level actions",
- Arrays.asList("AMBARI.ADMINISTRATOR:AMBARI", "CLUSTER.ADMINISTRATOR:CLUSTER"));
-
- addRoleAuthorization("AMBARI.RUN_CUSTOM_COMMAND", "Perform custom administrative actions",
- Collections.singletonList("AMBARI.ADMINISTRATOR:AMBARI"));
- }
-
- protected void updateTablesForMysql() throws SQLException {
- final Configuration.DatabaseType databaseType = configuration.getDatabaseType();
- if (databaseType == Configuration.DatabaseType.MYSQL) {
- dbAccessor.alterColumn(EXTENSION_TABLE, new DBAccessor.DBColumnInfo(EXTENSION_NAME_COLUMN, String.class, 100, null, false));
- dbAccessor.alterColumn(EXTENSION_TABLE, new DBAccessor.DBColumnInfo(EXTENSION_VERSION_COLUMN, String.class, 100, null, false));
-
- dbAccessor.alterColumn(USERS_TABLE, new DBAccessor.DBColumnInfo(USER_TYPE_COLUMN, String.class, 100, null, false));
- dbAccessor.alterColumn(USERS_TABLE, new DBAccessor.DBColumnInfo(USER_NAME_COLUMN, String.class, 100, null, false));
-
- dbAccessor.alterColumn(HOST_ROLE_COMMAND_TABLE, new DBAccessor.DBColumnInfo(ROLE_COLUMN, String.class, 100, null, true));
- dbAccessor.alterColumn(HOST_ROLE_COMMAND_TABLE, new DBAccessor.DBColumnInfo(STATUS_COLUMN, String.class, 100, null, true));
-
- dbAccessor.dropFKConstraint(HOST_GROUP_TABLE, "FK_hg_blueprint_name");
-
- dbAccessor.dropFKConstraint(HOST_GROUP_TABLE, "FK_hostgroup_blueprint_name");
-
- dbAccessor.dropFKConstraint(BLUEPRINT_CONFIGURATION, "FK_cfg_blueprint_name");
-
- dbAccessor.dropFKConstraint(BLUEPRINT_CONFIGURATION, "FK_blueprint_configuration_blueprint_name");
-
- dbAccessor.dropFKConstraint(BLUEPRINT_SETTING, "FK_blueprint_setting_blueprint_name");
-
- dbAccessor.dropFKConstraint(BLUEPRINT_SETTING, "FK_blueprint_setting_name");
-
- dbAccessor.alterColumn(BLUEPRINT_TABLE, new DBAccessor.DBColumnInfo(BLUEPRINT_NAME_COLUMN, String.class, 100, null, false));
-
- String[] uniqueColumns1 = new String[] { BLUEPRINT_NAME_COLUMN };
-
- dbAccessor.addFKConstraint(HOST_GROUP_TABLE, "FK_hg_blueprint_name",
- uniqueColumns1, BLUEPRINT_TABLE, uniqueColumns1, false);
-
- dbAccessor.addFKConstraint(BLUEPRINT_CONFIGURATION, "FK_cfg_blueprint_name",
- uniqueColumns1, BLUEPRINT_TABLE, uniqueColumns1, false);
-
- dbAccessor.addFKConstraint(BLUEPRINT_SETTING, "FK_blueprint_setting_name",
- uniqueColumns1, BLUEPRINT_TABLE, uniqueColumns1, false);
- }
- }
-
- /**
- * Convert the previously set inherited privileges to the more generic inherited privileges model
- * based on role-based principals rather than specialized principal types.
- */
- @Transactional
- void convertRolePrincipals() {
- LOG.info("Converting pseudo principle types to role principals");
-
- PermissionDAO permissionDAO = injector.getInstance(PermissionDAO.class);
- PrivilegeDAO privilegeDAO = injector.getInstance(PrivilegeDAO.class);
- PrincipalDAO principalDAO = injector.getInstance(PrincipalDAO.class);
- PrincipalTypeDAO principalTypeDAO = injector.getInstance(PrincipalTypeDAO.class);
-
- Map<String, String> principalTypeToRole = new HashMap<>();
- principalTypeToRole.put("ALL.CLUSTER.ADMINISTRATOR", "CLUSTER.ADMINISTRATOR");
- principalTypeToRole.put("ALL.CLUSTER.OPERATOR", "CLUSTER.OPERATOR");
- principalTypeToRole.put("ALL.CLUSTER.USER", "CLUSTER.USER");
- principalTypeToRole.put("ALL.SERVICE.ADMINISTRATOR", "SERVICE.ADMINISTRATOR");
- principalTypeToRole.put("ALL.SERVICE.OPERATOR", "SERVICE.OPERATOR");
-
- // Handle a typo introduced in org.apache.ambari.server.upgrade.UpgradeCatalog240.updateClusterInheritedPermissionsConfig
- principalTypeToRole.put("ALL.SERVICE.OPERATIOR", "SERVICE.OPERATOR");
-
- for (Map.Entry<String, String> entry : principalTypeToRole.entrySet()) {
- String principalTypeName = entry.getKey();
- String roleName = entry.getValue();
-
- PermissionEntity role = permissionDAO.findByName(roleName);
- PrincipalEntity rolePrincipalEntity = (role == null) ? null : role.getPrincipal();
-
- // Convert Privilege Records
- PrincipalTypeEntity principalTypeEntity = principalTypeDAO.findByName(principalTypeName);
-
- if (principalTypeEntity != null) {
- List<PrincipalEntity> principalEntities = principalDAO.findByPrincipalType(principalTypeName);
-
- for (PrincipalEntity principalEntity : principalEntities) {
- Set<PrivilegeEntity> privilegeEntities = principalEntity.getPrivileges();
-
- for (PrivilegeEntity privilegeEntity : privilegeEntities) {
- if (rolePrincipalEntity == null) {
- LOG.info("Removing privilege (id={}) since no role principle was found for {}:\n{}",
- privilegeEntity.getId(), roleName, formatPrivilegeEntityDetails(privilegeEntity));
- // Remove this privilege
- privilegeDAO.remove(privilegeEntity);
- } else {
- LOG.info("Updating privilege (id={}) to use role principle for {}:\n{}",
- privilegeEntity.getId(), roleName, formatPrivilegeEntityDetails(privilegeEntity));
-
- // Set the principal to the updated principal value
- privilegeEntity.setPrincipal(rolePrincipalEntity);
- privilegeDAO.merge(privilegeEntity);
- }
- }
-
- // Remove the obsolete principal
- principalDAO.remove(principalEntity);
- }
-
- // Remove the obsolete principal type
- principalTypeDAO.remove(principalTypeEntity);
- }
- }
-
- LOG.info("Converting pseudo principle types to role principals - complete.");
- }
-
- private String formatPrivilegeEntityDetails(PrivilegeEntity privilegeEntity) {
- if (privilegeEntity == null) {
- return "";
- } else {
- ResourceEntity resource = privilegeEntity.getResource();
- PrincipalEntity principal = privilegeEntity.getPrincipal();
- PermissionEntity permission = privilegeEntity.getPermission();
-
- return String.format("" +
- "\tPrivilege ID: %d" +
- "\n\tResource ID: %d" +
- "\n\tPrincipal ID: %d" +
- "\n\tPermission ID: %d",
- privilegeEntity.getId(),
- resource.getId(),
- principal.getId(),
- permission.getId()
- );
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/af1bf85c/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
deleted file mode 100644
index aed4adf..0000000
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ /dev/null
@@ -1,1352 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.server.upgrade;
-
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.ambari.server.AmbariException;
-import org.apache.ambari.server.actionmanager.CommandExecutionType;
-import org.apache.ambari.server.collections.Predicate;
-import org.apache.ambari.server.collections.functors.ContainsPredicate;
-import org.apache.ambari.server.configuration.Configuration;
-import org.apache.ambari.server.controller.AmbariManagementController;
-import org.apache.ambari.server.orm.DBAccessor;
-import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo;
-import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
-import org.apache.ambari.server.orm.dao.AlertsDAO;
-import org.apache.ambari.server.orm.dao.ArtifactDAO;
-import org.apache.ambari.server.orm.dao.DaoUtils;
-import org.apache.ambari.server.orm.dao.ViewInstanceDAO;
-import org.apache.ambari.server.orm.entities.AlertCurrentEntity;
-import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
-import org.apache.ambari.server.orm.entities.AlertHistoryEntity;
-import org.apache.ambari.server.orm.entities.ArtifactEntity;
-import org.apache.ambari.server.orm.entities.ViewInstanceEntity;
-import org.apache.ambari.server.state.Cluster;
-import org.apache.ambari.server.state.Clusters;
-import org.apache.ambari.server.state.Config;
-import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor;
-import org.apache.ambari.server.state.kerberos.KerberosConfigurationDescriptor;
-import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
-import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
-import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor;
-import org.apache.ambari.server.state.kerberos.KerberosKeytabDescriptor;
-import org.apache.ambari.server.state.kerberos.KerberosPrincipalDescriptor;
-import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
-import org.apache.ambari.server.view.ViewArchiveUtility;
-import org.apache.ambari.server.view.ViewInstanceOperationHandler;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.jdbc.support.JdbcUtils;
-
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.google.gson.JsonPrimitive;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-
-/**
- * Upgrade catalog for version 2.5.0.
- */
-public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
-
- protected static final String HOST_VERSION_TABLE = "host_version";
- protected static final String GROUPS_TABLE = "groups";
- protected static final String GROUP_TYPE_COL = "group_type";
- private static final String AMS_ENV = "ams-env";
- private static final String AMS_GRAFANA_INI = "ams-grafana-ini";
- private static final String AMS_SITE = "ams-site";
- private static final String AMS_LOG4J = "ams-log4j";
- private static final String AMS_HBASE_LOG4J = "ams-hbase-log4j";
- private static final String AMS_MODE = "timeline.metrics.service.operation.mode";
- private static final String AMS_HBASE_SITE = "ams-hbase-site";
- private static final String HBASE_ROOTDIR = "hbase.rootdir";
- private static final String HADOOP_ENV = "hadoop-env";
- private static final String KAFKA_BROKER = "kafka-broker";
- private static final String YARN_SITE_CONFIG = "yarn-site";
- private static final String YARN_ENV_CONFIG = "yarn-env";
- private static final String YARN_LCE_CGROUPS_MOUNT_PATH = "yarn.nodemanager.linux-container-executor.cgroups.mount-path";
- private static final String YARN_CGROUPS_ENABLED = "yarn_cgroups_enabled";
- private static final String KAFKA_TIMELINE_METRICS_HOST = "kafka.timeline.metrics.host";
-
- public static final String COMPONENT_TABLE = "servicecomponentdesiredstate";
- public static final String COMPONENT_VERSION_TABLE = "servicecomponent_version";
- public static final String COMPONENT_VERSION_PK = "PK_sc_version";
- public static final String COMPONENT_VERSION_FK_COMPONENT = "FK_scv_component_id";
- public static final String COMPONENT_VERSION_FK_REPO_VERSION = "FK_scv_repo_version_id";
-
- protected static final String SERVICE_DESIRED_STATE_TABLE = "servicedesiredstate";
- protected static final String CREDENTIAL_STORE_ENABLED_COL = "credential_store_enabled";
-
- protected static final String HOST_COMPONENT_DESIREDSTATE_TABLE = "hostcomponentdesiredstate";
- protected static final String HOST_COMPONENT_DESIREDSTATE_ID_COL = "id";
- protected static final String HOST_COMPONENT_DESIREDSTATE_INDEX = "UQ_hcdesiredstate_name";
-
- @Inject
- ViewInstanceDAO viewInstanceDAO;
-
- @Inject
- ViewInstanceOperationHandler viewInstanceOperationHandler;
-
- @Inject
- protected ViewArchiveUtility archiveUtility;
-
- /**
- * Logger.
- */
- private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog250.class);
-
- @Inject
- DaoUtils daoUtils;
-
- // ----- Constructors ------------------------------------------------------
-
- /**
- * Don't forget to register new UpgradeCatalogs in {@link org.apache.ambari.server.upgrade.SchemaUpgradeHelper.UpgradeHelperModule#configure()}
- *
- * @param injector Guice injector to track dependencies and uses bindings to inject them.
- */
- @Inject
- public UpgradeCatalog250(Injector injector) {
- super(injector);
-
- daoUtils = injector.getInstance(DaoUtils.class);
- }
-
- // ----- UpgradeCatalog ----------------------------------------------------
-
- /**
- * {@inheritDoc}
- */
- @Override
- public String getTargetVersion() {
- return "2.5.0";
- }
-
- // ----- AbstractUpgradeCatalog --------------------------------------------
-
- /**
- * {@inheritDoc}
- */
- @Override
- public String getSourceVersion() {
- return "2.4.2";
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- protected void executeDDLUpdates() throws AmbariException, SQLException {
- updateHostVersionTable();
- createComponentVersionTable();
- updateGroupsTable();
- dbAccessor.addColumn("stage",
- new DBAccessor.DBColumnInfo("command_execution_type", String.class, 32, CommandExecutionType.STAGE.toString(),
- false));
- updateServiceDesiredStateTable();
- updateHostComponentDesiredStateTable();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- protected void executePreDMLUpdates() throws AmbariException, SQLException {
-
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- protected void executeDMLUpdates() throws AmbariException, SQLException {
- addNewConfigurationsFromXml();
- updateAMSConfigs();
- updateStormAlerts();
- updateLogSearchAlert();
- removeAlertDuplicates();
- updateHadoopEnvConfigs();
- updateKafkaConfigs();
- updateHIVEInteractiveConfigs();
- unInstallAllZeppelinViews();
- updateZeppelinConfigs();
- updateAtlasConfigs();
- updateLogSearchConfigs();
- updateAmbariInfraConfigs();
- updateYarnSite();
- updateRangerUrlConfigs();
- addManageServiceAutoStartPermissions();
- addManageAlertNotificationsPermissions();
- updateKerberosDescriptorArtifacts();
- fixHBaseMasterCPUUtilizationAlertDefinition();
- updateTezHistoryUrlBase();
- }
-
- /**
- * Fix the HBase Master CPU Utilization alert definition by swapping the values for <code>kerberos_keytab</code>
- * and <code>kerberos_principal</code>.
- */
- protected void fixHBaseMasterCPUUtilizationAlertDefinition() {
- AlertDefinitionDAO alertDefinitionDAO = injector.getInstance(AlertDefinitionDAO.class);
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
-
- Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
- for (final Cluster cluster : clusterMap.values()) {
- long clusterID = cluster.getClusterId();
- AlertDefinitionEntity alertDefinition = alertDefinitionDAO.findByName(clusterID, "hbase_master_cpu");
- if(alertDefinition != null) {
- LOG.info("Updating alert definition {} in cluster {}", alertDefinition.getDefinitionName(), clusterID);
- String source = alertDefinition.getSource();
-
- if(source != null) {
- JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject();
-
- if(sourceJson != null) {
- boolean changesExist = false;
- LOG.debug("Source before update : {}", sourceJson);
-
- JsonObject uriJson = sourceJson.get("uri").getAsJsonObject();
- JsonPrimitive primitive;
-
- if (uriJson != null) {
- // Replace
- // "kerberos_keytab": "{{hbase-site/hbase.security.authentication.spnego.kerberos.principal}}"
- // With
- // "kerberos_keytab": "{{hbase-site/hbase.security.authentication.spnego.kerberos.keytab}}"
- primitive = uriJson.getAsJsonPrimitive("kerberos_keytab");
- if ((primitive != null) && primitive.isString() && "{{hbase-site/hbase.security.authentication.spnego.kerberos.principal}}".equals(primitive.getAsString())) {
- uriJson.remove("kerberos_keytab");
- uriJson.addProperty("kerberos_keytab", "{{hbase-site/hbase.security.authentication.spnego.kerberos.keytab}}");
- changesExist = true;
- }
-
- // Replace
- // "kerberos_principal": "{{hbase-site/hbase.security.authentication.spnego.kerberos.keytab}}"
- // With
- // "kerberos_principal": "{{hbase-site/hbase.security.authentication.spnego.kerberos.principal}}"
- primitive = uriJson.getAsJsonPrimitive("kerberos_principal");
- if ((primitive != null) && primitive.isString() && "{{hbase-site/hbase.security.authentication.spnego.kerberos.keytab}}".equals(primitive.getAsString())) {
- uriJson.remove("kerberos_principal");
- uriJson.addProperty("kerberos_principal", "{{hbase-site/hbase.security.authentication.spnego.kerberos.principal}}");
- changesExist = true;
- }
- }
-
- LOG.debug("Source after update : {}", sourceJson);
- if(changesExist) {
- alertDefinition.setSource(sourceJson.toString());
- alertDefinition.setHash(UUID.randomUUID().toString());
-
- alertDefinitionDAO.merge(alertDefinition);
- }
- }
- }
- }
- }
- }
-
- /**
- * Removes all {@link AlertCurrentEntity} duplicates from database.
- * Alerts are considered as duplicates if their definition, host and alert instance are the same.
- * Duplicates could be created in earlier versions of Ambari up till 2.4.1.
- */
- protected void removeAlertDuplicates() {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- AlertsDAO alertsDao = injector.getInstance(AlertsDAO.class);
- Clusters clusters = ambariManagementController.getClusters();
-
- Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
- for (final Cluster cluster : clusterMap.values()) {
- long clusterID = cluster.getClusterId();
- LOG.info("Removing alert duplicates on cluster {}", cluster.getClusterName());
- List<AlertCurrentEntity> alertCurrentEntities = alertsDao.findCurrentByCluster(clusterID);
- Set<AlertHistoryEntity> uniqueAlerts = new HashSet<>();
- for (AlertCurrentEntity alertCurrentEntity : alertCurrentEntities) {
-
- AlertHistoryEntity currentAlert = new AlertHistoryEntity();
- currentAlert.setAlertDefinition(alertCurrentEntity.getAlertHistory().getAlertDefinition());
- currentAlert.setHostName(alertCurrentEntity.getAlertHistory().getHostName());
- currentAlert.setAlertInstance(alertCurrentEntity.getAlertHistory().getAlertInstance());
-
- if (uniqueAlerts.contains(currentAlert)) {
- LOG.info("Alert entity duplicate {} will be removed",alertCurrentEntity.getAlertHistory());
- alertsDao.remove(alertCurrentEntity);
- } else {
- uniqueAlerts.add(currentAlert);
- }
- }
- }
- }
-
- protected void updateStormAlerts() {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- AlertDefinitionDAO alertDefinitionDAO = injector.getInstance(AlertDefinitionDAO.class);
- Clusters clusters = ambariManagementController.getClusters();
-
- Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
- for (final Cluster cluster : clusterMap.values()) {
- long clusterID = cluster.getClusterId();
- LOG.info("Updating storm alert definitions on cluster : " + cluster.getClusterName());
-
- final AlertDefinitionEntity stormServerProcessDefinitionEntity = alertDefinitionDAO.findByName(
- clusterID, "storm_server_process");
-
- final AlertDefinitionEntity stormWebAlert = alertDefinitionDAO.findByName(
- clusterID, "storm_webui");
-
- if (stormServerProcessDefinitionEntity != null) {
- LOG.info("Removing alert definition : " + stormServerProcessDefinitionEntity);
- alertDefinitionDAO.remove(stormServerProcessDefinitionEntity);
- }
-
- if (stormWebAlert != null) {
- LOG.info("Updating alert definition : " + stormWebAlert.getDefinitionName());
- String source = stormWebAlert.getSource();
- JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject();
- LOG.debug("Source before update : {}", sourceJson);
-
- JsonObject uriJson = sourceJson.get("uri").getAsJsonObject();
- uriJson.remove("https");
- uriJson.remove("https_property");
- uriJson.remove("https_property_value");
- uriJson.addProperty("https", "{{storm-site/ui.https.port}}");
- uriJson.addProperty("https_property", "{{storm-site/ui.https.keystore.type}}");
- uriJson.addProperty("https_property_value", "jks");
-
- LOG.debug("Source after update : {}", sourceJson);
- stormWebAlert.setSource(sourceJson.toString());
- alertDefinitionDAO.merge(stormWebAlert);
- }
- }
- }
-
- protected void updateLogSearchAlert() {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- AlertDefinitionDAO alertDefinitionDAO = injector.getInstance(AlertDefinitionDAO.class);
- Clusters clusters = ambariManagementController.getClusters();
-
- Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
- for (final Cluster cluster : clusterMap.values()) {
- long clusterID = cluster.getClusterId();
- LOG.info("Updating Log Search web ui alert definitions on cluster : " + cluster.getClusterName());
-
- final AlertDefinitionEntity logSearchWebAlert = alertDefinitionDAO.findByName(
- clusterID, "logsearch_ui");
-
- if (logSearchWebAlert != null) {
- LOG.info("Updating alert definition : " + logSearchWebAlert.getDefinitionName());
- String source = logSearchWebAlert.getSource();
- JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject();
- LOG.debug("Source before update : {}", sourceJson);
-
- JsonObject uriJson = sourceJson.get("uri").getAsJsonObject();
- uriJson.remove("https_property");
- uriJson.remove("https_property_value");
- uriJson.addProperty("https_property", "{{logsearch-env/logsearch_ui_protocol}}");
- uriJson.addProperty("https_property_value", "https");
-
- LOG.debug("Source after update : {}", sourceJson);
- logSearchWebAlert.setSource(sourceJson.toString());
- alertDefinitionDAO.merge(logSearchWebAlert);
- }
- }
- }
-
- /**
- * This will check if previous value of 'tez.tez-ui.history-url.base' contains tez view's url.
- * If yes then it will point it to fixed url of tez view auto view as introduced in ambari-2.5.0.0.
- * else it will log an error and move ahead.
- */
- protected void updateTezHistoryUrlBase() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
-
- if (clusters != null) {
- Map<String, Cluster> clusterMap = clusters.getClusters();
- if (clusterMap != null && !clusterMap.isEmpty()) {
- for (final Cluster cluster : clusterMap.values()) {
- Set<String> installedServices = cluster.getServices().keySet();
- if (installedServices.contains("TEZ")) {
- Config tezSite = cluster.getDesiredConfigByType("tez-site");
- if (tezSite != null) {
- String currentTezHistoryUrlBase = tezSite.getProperties().get("tez.tez-ui.history-url.base");
- if (!StringUtils.isEmpty(currentTezHistoryUrlBase)) {
- LOG.info("Current Tez History URL base: {} ", currentTezHistoryUrlBase);
- String newTezHistoryUrlBase = null;
- try {
- newTezHistoryUrlBase = getUpdatedTezHistoryUrlBase(currentTezHistoryUrlBase);
- } catch (AmbariException e) {
- LOG.error("Error occurred while creating updated URL of tez view using value in property tez.tez-ui.history-url.base." +
- "The current value {} is not of standard format expected by Ambari. Skipping the updation of tez.tez-ui.history-url.base." +
- "Please check validity of this property manually in tez site after upgrade.", currentTezHistoryUrlBase, e);
- return;
- }
- LOG.info("New Tez History URL base: {} ", newTezHistoryUrlBase);
- updateConfigurationProperties("tez-site", Collections.singletonMap("tez.tez-ui.history-url.base", newTezHistoryUrlBase), true, false);
- }
- }
- }
- }
- }
- }
- }
-
- /**
- * Transforms the existing tez history url base to the fixed short url for tez auto instance
- * @param currentTezHistoryUrlBase Existing value of the tez history url base
- * @return the updated tez history url base
- * @throws AmbariException if currentTezHistoryUrlBase is malformed or is not compatible with the Tez View url REGEX
- */
- protected String getUpdatedTezHistoryUrlBase(String currentTezHistoryUrlBase) throws AmbariException{
- String pattern = "(.*)(\\/views\\/TEZ\\/)(.*)";
- Pattern regex = Pattern.compile(pattern);
- Matcher matcher = regex.matcher(currentTezHistoryUrlBase);
- String prefix;
- if (matcher.find()) {
- prefix = matcher.group(1);
- } else {
- throw new AmbariException("Cannot prepare the new value for property: 'tez.tez-ui.history-url.base' using the old value: '" + currentTezHistoryUrlBase + "'");
- }
-
- // adding the auto tez instance short url name instead of the tez version and tez view instance name
- return prefix + "/view/TEZ/tez_cluster_instance";
- }
-
- protected void updateHostVersionTable() throws SQLException {
- LOG.info("Updating the {} table", HOST_VERSION_TABLE);
-
- // Add the unique constraint to the host_version table
- dbAccessor.addUniqueConstraint(HOST_VERSION_TABLE, "UQ_host_repo", "repo_version_id", "host_id");
- }
-
- protected void updateGroupsTable() throws SQLException {
- LOG.info("Updating the {} table", GROUPS_TABLE);
-
- dbAccessor.addColumn(GROUPS_TABLE, new DBColumnInfo(GROUP_TYPE_COL, String.class, null, "LOCAL", false));
- dbAccessor.executeQuery("UPDATE groups SET group_type='LDAP' WHERE ldap_group=1");
- dbAccessor.addUniqueConstraint(GROUPS_TABLE, "UNQ_groups_0", "group_name", "group_type");
- }
-
- /**
- * Updates {@code yarn-site} in the following ways:
- *
- * Remove {@code YARN_LCE_CGROUPS_MOUNT_PATH} if {@code YARN_CGROUPS_ENABLED} is {@code false} and
- * {@code YARN_LCE_CGROUPS_MOUNT_PATH} is empty string
- *
- * @throws AmbariException
- */
- protected void updateYarnSite() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
- Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
-
- for (final Cluster cluster : clusterMap.values()) {
- Config yarnEnvConfig = cluster.getDesiredConfigByType(YARN_ENV_CONFIG);
- Config yarnSiteConfig = cluster.getDesiredConfigByType(YARN_SITE_CONFIG);
-
- if (yarnEnvConfig != null && yarnSiteConfig != null) {
- String cgroupEnabled = yarnEnvConfig.getProperties().get(YARN_CGROUPS_ENABLED);
- String mountPath = yarnSiteConfig.getProperties().get(YARN_LCE_CGROUPS_MOUNT_PATH);
-
- if (StringUtils.isEmpty(mountPath) && cgroupEnabled != null
- && cgroupEnabled.trim().equalsIgnoreCase("false")){
-
- removeConfigurationPropertiesFromCluster(cluster, YARN_SITE_CONFIG, new HashSet<String>(){{
- add(YARN_LCE_CGROUPS_MOUNT_PATH);
- }});
-
- }
- }
-
- }
- }
-
- protected void updateAMSConfigs() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
-
- if (clusters != null) {
- Map<String, Cluster> clusterMap = clusters.getClusters();
-
- if (clusterMap != null && !clusterMap.isEmpty()) {
- for (final Cluster cluster : clusterMap.values()) {
-
- Config amsEnv = cluster.getDesiredConfigByType(AMS_ENV);
- if (amsEnv != null) {
- Map<String, String> amsEnvProperties = amsEnv.getProperties();
- String content = amsEnvProperties.get("content");
- Map<String, String> newProperties = new HashMap<>();
- newProperties.put("content", updateAmsEnvContent(content));
- updateConfigurationPropertiesForCluster(cluster, AMS_ENV, newProperties, true, true);
- }
-
-
- boolean isDistributed = false;
- Config amsSite = cluster.getDesiredConfigByType(AMS_SITE);
- if (amsSite != null) {
- if ("distributed".equals(amsSite.getProperties().get(AMS_MODE))) {
- isDistributed = true;
- }
-
- Map<String, String> amsSiteProperties = amsSite.getProperties();
-
- if (amsSiteProperties != null && amsSiteProperties.containsKey("timeline.metrics.hbase.fifo.compaction.enabled")) {
- LOG.info("Removing timeline.metrics.hbase.fifo.compaction.enabled from ams-site");
- removeConfigurationPropertiesFromCluster(cluster, AMS_SITE, Collections.singleton("timeline.metrics.hbase.fifo.compaction.enabled"));
- }
- }
-
- if (isDistributed) {
- Config amsHbaseSite = cluster.getDesiredConfigByType(AMS_HBASE_SITE);
- if (amsHbaseSite != null) {
- Map<String, String> amsHbaseSiteProperties = amsHbaseSite.getProperties();
- String rootDir = amsHbaseSiteProperties.get(HBASE_ROOTDIR);
- if (StringUtils.isNotEmpty(rootDir) && rootDir.startsWith("hdfs://")) {
- int indexOfSlash = rootDir.indexOf("/", 7);
- Map<String, String> newProperties = new HashMap<>();
- String newRootdir = rootDir.substring(indexOfSlash);
- newProperties.put(HBASE_ROOTDIR, newRootdir);
- LOG.info("Changing ams-hbase-site rootdir to " + newRootdir);
- updateConfigurationPropertiesForCluster(cluster, AMS_HBASE_SITE, newProperties, true, true);
- }
- }
- }
-
- //Update AMS log4j to make rolling properties configurable as separate fields.
- Config amsLog4jProperties = cluster.getDesiredConfigByType(AMS_LOG4J);
- if(amsLog4jProperties != null){
- Map<String, String> newProperties = new HashMap<>();
-
- String content = amsLog4jProperties.getProperties().get("content");
- content = SchemaUpgradeUtil.extractProperty(content,"ams_log_max_backup_size","ams_log_max_backup_size","log4j.appender.file.MaxFileSize=(\\w+)MB","80",newProperties);
- content = SchemaUpgradeUtil.extractProperty(content,"ams_log_number_of_backup_files","ams_log_number_of_backup_files","log4j.appender.file.MaxBackupIndex=(\\w+)","60",newProperties);
- newProperties.put("content",content);
- updateConfigurationPropertiesForCluster(cluster,AMS_LOG4J,newProperties,true,true);
- }
-
- Config amsHbaseLog4jProperties = cluster.getDesiredConfigByType(AMS_HBASE_LOG4J);
- if(amsHbaseLog4jProperties != null){
- Map<String, String> newProperties = new HashMap<>();
-
- String content = amsHbaseLog4jProperties.getProperties().get("content");
- content = SchemaUpgradeUtil.extractProperty(content,"ams_hbase_log_maxfilesize","ams_hbase_log_maxfilesize","hbase.log.maxfilesize=(\\w+)MB","256",newProperties);
- content = SchemaUpgradeUtil.extractProperty(content,"ams_hbase_log_maxbackupindex","ams_hbase_log_maxbackupindex","hbase.log.maxbackupindex=(\\w+)","20",newProperties);
- content = SchemaUpgradeUtil.extractProperty(content,"ams_hbase_security_log_maxfilesize","ams_hbase_security_log_maxfilesize","hbase.security.log.maxfilesize=(\\w+)MB","256",newProperties);
- content = SchemaUpgradeUtil.extractProperty(content,"ams_hbase_security_log_maxbackupindex","ams_hbase_security_log_maxbackupindex","hbase.security.log.maxbackupindex=(\\w+)","20",newProperties);
- newProperties.put("content",content);
- updateConfigurationPropertiesForCluster(cluster,AMS_HBASE_LOG4J,newProperties,true,true);
- }
-
- Config amsGrafanaIni = cluster.getDesiredConfigByType(AMS_GRAFANA_INI);
- if (amsGrafanaIni != null) {
- Map<String, String> amsGrafanaIniProperties = amsGrafanaIni.getProperties();
- String content = amsGrafanaIniProperties.get("content");
- Map<String, String> newProperties = new HashMap<>();
- newProperties.put("content", updateAmsGrafanaIniContent(content));
- updateConfigurationPropertiesForCluster(cluster, AMS_GRAFANA_INI, newProperties, true, true);
- }
-
- }
- }
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- protected void updateKerberosDescriptorArtifact(ArtifactDAO artifactDAO, ArtifactEntity artifactEntity) throws AmbariException {
- if (artifactEntity != null) {
- Map<String, Object> data = artifactEntity.getArtifactData();
-
- if (data != null) {
- final KerberosDescriptor kerberosDescriptor = new KerberosDescriptorFactory().createInstance(data);
-
- if (kerberosDescriptor != null) {
- KerberosServiceDescriptor logSearchKerberosDescriptor = kerberosDescriptor.getService("LOGSEARCH");
- KerberosServiceDescriptor atlasKerberosDescriptor = kerberosDescriptor.getService("ATLAS");
- KerberosServiceDescriptor rangerKerberosDescriptor = kerberosDescriptor.getService("RANGER");
- addInfrSolrDescriptor(artifactDAO, artifactEntity, kerberosDescriptor, atlasKerberosDescriptor, "ATLAS_SERVER");
- addInfrSolrDescriptor(artifactDAO, artifactEntity, kerberosDescriptor, logSearchKerberosDescriptor, "LOGSEARCH_SERVER");
- addInfrSolrDescriptor(artifactDAO, artifactEntity, kerberosDescriptor, rangerKerberosDescriptor, "RANGER_ADMIN");
- KerberosServiceDescriptor stormKerberosDescriptor = kerberosDescriptor.getService("STORM");
-
- if (stormKerberosDescriptor != null) {
- KerberosComponentDescriptor componentDescriptor = stormKerberosDescriptor.getComponent("NIMBUS");
- if (componentDescriptor != null) {
- KerberosIdentityDescriptor origIdentityDescriptor = componentDescriptor.getIdentity("/STORM/NIMBUS/nimbus_server");
- if (origIdentityDescriptor != null) {
- KerberosPrincipalDescriptor origPrincipalDescriptor = origIdentityDescriptor.getPrincipalDescriptor();
- KerberosPrincipalDescriptor newPrincipalDescriptor = new KerberosPrincipalDescriptor(
- null,
- null,
- (origPrincipalDescriptor == null) ?
- "ranger-storm-audit/xasecure.audit.jaas.Client.option.principal" : origPrincipalDescriptor.getConfiguration(),
- null
- );
- KerberosKeytabDescriptor origKeytabDescriptor = origIdentityDescriptor.getKeytabDescriptor();
- KerberosKeytabDescriptor newKeytabDescriptor = new KerberosKeytabDescriptor(
- null,
- null,
- null,
- null,
- null,
- (origKeytabDescriptor == null) ?
- "ranger-storm-audit/xasecure.audit.jaas.Client.option.keyTab" : origKeytabDescriptor.getConfiguration(),
- false);
- componentDescriptor.removeIdentity("/STORM/NIMBUS/nimbus_server");
- componentDescriptor.putIdentity(new KerberosIdentityDescriptor("/STORM/storm_components", null, newPrincipalDescriptor, newKeytabDescriptor, null));
- }
- }
- }
- KerberosServiceDescriptor yarnKerberosDescriptor = kerberosDescriptor.getService("YARN");
- if (yarnKerberosDescriptor != null) {
- Map<String, KerberosConfigurationDescriptor> configs = yarnKerberosDescriptor.getConfigurations();
- KerberosConfigurationDescriptor yarnSiteConfigDescriptor = configs.get("yarn-site");
- if (yarnSiteConfigDescriptor != null) {
- Map<String, String> properties = yarnSiteConfigDescriptor.getProperties();
- if (properties != null && properties.containsKey(YARN_LCE_CGROUPS_MOUNT_PATH)) {
- properties.remove(YARN_LCE_CGROUPS_MOUNT_PATH);
- }
- }
- }
-
- // Fix HBASE_MASTER Kerberos identity for Ranger audit by clearing out any keytab file or principal name values.
- KerberosServiceDescriptor hbaseKerberosDescriptor = kerberosDescriptor.getService("HBASE");
- if (hbaseKerberosDescriptor != null) {
- KerberosComponentDescriptor hbaseMasterKerberosDescriptor = hbaseKerberosDescriptor.getComponent("HBASE_MASTER");
- if (hbaseMasterKerberosDescriptor != null) {
- KerberosIdentityDescriptor identityDescriptor = hbaseMasterKerberosDescriptor.getIdentity("/HBASE/HBASE_MASTER/hbase_master_hbase");
-
- if (identityDescriptor != null) {
- KerberosPrincipalDescriptor principalDescriptor = identityDescriptor.getPrincipalDescriptor();
- KerberosKeytabDescriptor keytabDescriptor = identityDescriptor.getKeytabDescriptor();
-
- identityDescriptor.setReference(identityDescriptor.getName());
- identityDescriptor.setName("ranger_hbase_audit");
-
- principalDescriptor.setValue(null);
- keytabDescriptor.setFile(null);
- }
- }
- }
-
- artifactEntity.setArtifactData(kerberosDescriptor.toMap());
- artifactDAO.merge(artifactEntity);
- }
- }
- }
- }
-
- /**
- * Add /AMBARI-INFRA/INFRA_SOLR/infra-solr reference to specific service component
- */
- private void addInfrSolrDescriptor(ArtifactDAO artifactDAO, ArtifactEntity artifactEntity, KerberosDescriptor kerberosDescriptor,
- KerberosServiceDescriptor serviceDescriptor, String componentName) {
- if (serviceDescriptor != null) {
- KerberosComponentDescriptor componentDescriptor = serviceDescriptor.getComponent(componentName);
- if (componentDescriptor != null) {
- KerberosIdentityDescriptor origIdentityDescriptor = componentDescriptor.getIdentity("/AMBARI_INFRA/INFRA_SOLR/infra-solr");
- if (origIdentityDescriptor != null) {
- LOG.info("/AMBARI_INFRA/INFRA_SOLR/infra-solr identity already exists in {} component", componentName);
- } else {
- Predicate predicate = ContainsPredicate.fromMap(Collections.<String, Object>singletonMap(ContainsPredicate.NAME, Arrays.asList("services", "AMBARI_INFRA")));
- componentDescriptor.putIdentity(new KerberosIdentityDescriptor("/AMBARI_INFRA/INFRA_SOLR/infra-solr",null, null, null, predicate));
- }
- }
- }
- }
-
- protected void unInstallAllZeppelinViews(){
- LOG.info("Removing all Zeppelin views.");
- List<ViewInstanceEntity> viewInstanceList = viewInstanceDAO.findAll();
- for( ViewInstanceEntity viewInstanceEntity : viewInstanceList ){
- if(viewInstanceEntity.getViewName().equalsIgnoreCase("ZEPPELIN{1.0.0}")){
- LOG.info("Uninstalling zeppelin view : {}", viewInstanceEntity);
- try {
- viewInstanceOperationHandler.uninstallViewInstance(viewInstanceEntity);
- }catch(Exception e){
- LOG.error("Exception occurred while uninstalling view {}. Ignored for now.", viewInstanceEntity);
- }
- }
- }
- }
-
- /**
- * Updates Zeppelin configs.
- *
- * @throws AmbariException
- */
- protected void updateZeppelinConfigs() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
- if (clusters != null) {
- Map<String, Cluster> clusterMap = clusters.getClusters();
-
- if (clusterMap != null && !clusterMap.isEmpty()) {
- for (final Cluster cluster : clusterMap.values()) {
- Config zeppelinEnvProperties = cluster.getDesiredConfigByType("zeppelin-env");
- if (zeppelinEnvProperties != null) {
- String log4jPropertiesContent = zeppelinEnvProperties.getProperties().get("log4j_properties_content");
- String shiroIniContent = zeppelinEnvProperties.getProperties().get("shiro_ini_content");
-
- updateConfigurationProperties("zeppelin-log4j-properties", Collections.singletonMap("log4j_properties_content", log4jPropertiesContent), true, true);
- updateConfigurationProperties("zeppelin-shiro-ini", Collections.singletonMap("shiro_ini_content", shiroIniContent), true, true);
- }
- }
- }
- }
- }
-
- protected String updateAmsGrafanaIniContent(String content) {
- if (content == null) {
- return null;
- }
-
- String toReplace = "admin_password = {{ams_grafana_admin_pwd}}";
- String replaceWith = ";admin_password =";
- content = content.replace(toReplace, replaceWith);
- return content;
- }
-
- protected String updateAmsEnvContent(String content) {
- if (content == null) {
- return null;
- }
-
- List<String> toReplaceList = new ArrayList<>();
- toReplaceList.add("\n# HBase normalizer enabled\n");
- toReplaceList.add("\n# HBase compaction policy enabled\n");
- toReplaceList.add("export AMS_HBASE_NORMALIZER_ENABLED={{ams_hbase_normalizer_enabled}}\n");
- toReplaceList.add("export AMS_HBASE_FIFO_COMPACTION_ENABLED={{ams_hbase_fifo_compaction_enabled}}\n");
-
- //Because of AMBARI-15331 : AMS HBase FIFO compaction policy and Normalizer settings are not handled correctly
- toReplaceList.add("export HBASE_NORMALIZATION_ENABLED={{ams_hbase_normalizer_enabled}}\n");
- toReplaceList.add("export HBASE_FIFO_COMPACTION_POLICY_ENABLED={{ams_hbase_fifo_compaction_policy_enabled}}\n");
-
-
- for (String toReplace : toReplaceList) {
- if (content.contains(toReplace)) {
- content = content.replace(toReplace, StringUtils.EMPTY);
- }
- }
-
- return content;
- }
-
- protected void updateKafkaConfigs() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
-
- if (clusters != null) {
- Map<String, Cluster> clusterMap = clusters.getClusters();
-
- if (clusterMap != null && !clusterMap.isEmpty()) {
- for (final Cluster cluster : clusterMap.values()) {
-
- Config kafkaBrokerConfig = cluster.getDesiredConfigByType(KAFKA_BROKER);
- if (kafkaBrokerConfig != null) {
- Map<String, String> kafkaBrokerProperties = kafkaBrokerConfig.getProperties();
-
- if (kafkaBrokerProperties != null && kafkaBrokerProperties.containsKey(KAFKA_TIMELINE_METRICS_HOST)) {
- LOG.info("Removing kafka.timeline.metrics.host from kafka-broker");
- removeConfigurationPropertiesFromCluster(cluster, KAFKA_BROKER, Collections.singleton("kafka.timeline.metrics.host"));
- }
- }
- }
- }
- }
- }
-
-
- protected void updateHadoopEnvConfigs() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(
- AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
-
- if (clusters != null) {
- Map<String, Cluster> clusterMap = clusters.getClusters();
- Map<String, String> prop = new HashMap<>();
-
-
- if (clusterMap != null && !clusterMap.isEmpty()) {
- for (final Cluster cluster : clusterMap.values()) {
-
- String content = null;
- Boolean contentUpdated = false;
-
- if (cluster.getDesiredConfigByType(HADOOP_ENV) != null) {
- content = cluster.getDesiredConfigByType(HADOOP_ENV).getProperties().get("content");
- }
-
- if (content != null) {
- if (!content.contains("ulimit -l")) { // Append "ulimit -l" to hadoop-env.sh
- content += "\n" +
- "{% if is_datanode_max_locked_memory_set %}\n" +
- "# Fix temporary bug, when ulimit from conf files is not picked up, without full relogin. \n" +
- "# Makes sense to fix only when runing DN as root \n" +
- "if [ \"$command\" == \"datanode\" ] && [ \"$EUID\" -eq 0 ] && [ -n \"$HADOOP_SECURE_DN_USER\" ]; then\n" +
- " ulimit -l {{datanode_max_locked_memory}}\n" +
- "fi\n" +
- "{% endif %}";
-
- contentUpdated = true;
- }
-
- if (!content.contains("ulimit -n")){ // Append "ulimit -n" to hadoop-env.sh
- content += "\n" +
- "if [ \"$command\" == \"datanode\" ] && [ \"$EUID\" -eq 0 ] && [ -n \"$HADOOP_SECURE_DN_USER\" ]; then \n" +
- " ulimit -n {{hdfs_user_nofile_limit}}\n" +
- "fi";
- contentUpdated = true;
- }
-
- if (contentUpdated){
- prop.put("content", content);
- updateConfigurationPropertiesForCluster(cluster, "hadoop-env",
- prop, true, false);
- }
- }
- }
- }
- }
- }
-
- /**
- * Creates the servicecomponent_version table
- * @throws SQLException
- */
- private void createComponentVersionTable() throws SQLException {
-
- List<DBColumnInfo> columns = new ArrayList<>();
-
- // Add extension link table
- LOG.info("Creating {} table", COMPONENT_VERSION_TABLE);
-
- columns.add(new DBColumnInfo("id", Long.class, null, null, false));
- columns.add(new DBColumnInfo("component_id", Long.class, null, null, false));
- columns.add(new DBColumnInfo("repo_version_id", Long.class, null, null, false));
- columns.add(new DBColumnInfo("state", String.class, 32, null, false));
- columns.add(new DBColumnInfo("user_name", String.class, 255, null, false));
- dbAccessor.createTable(COMPONENT_VERSION_TABLE, columns, (String[]) null);
-
- dbAccessor.addPKConstraint(COMPONENT_VERSION_TABLE, COMPONENT_VERSION_PK, "id");
-
- dbAccessor.addFKConstraint(COMPONENT_VERSION_TABLE, COMPONENT_VERSION_FK_COMPONENT, "component_id",
- COMPONENT_TABLE, "id", false);
-
- dbAccessor.addFKConstraint(COMPONENT_VERSION_TABLE, COMPONENT_VERSION_FK_REPO_VERSION, "repo_version_id",
- "repo_version", "repo_version_id", false);
-
- addSequence("servicecomponent_version_id_seq", 0L, false);
- }
-
- /**
- * Alter servicedesiredstate table.
- * @throws SQLException
- */
- private void updateServiceDesiredStateTable() throws SQLException {
- // ALTER TABLE servicedesiredstate ADD COLUMN
- // credential_store_enabled SMALLINT DEFAULT 0 NOT NULL
- dbAccessor.addColumn(SERVICE_DESIRED_STATE_TABLE,
- new DBColumnInfo(CREDENTIAL_STORE_ENABLED_COL, Short.class, null, 0, false));
- }
-
-
- /**
- * Removes the compound PK from hostcomponentdesiredstate table
- * and replaces it with a surrogate PK, but only if the table doesn't have it's new PK set.
- * Create index and unqiue constraint on the columns that originally formed the compound PK.
- *
- * @throws SQLException
- */
- private void updateHostComponentDesiredStateTable() throws SQLException {
- if (dbAccessor.tableHasPrimaryKey(HOST_COMPONENT_DESIREDSTATE_TABLE, HOST_COMPONENT_DESIREDSTATE_ID_COL)) {
- LOG.info("Skipping {} table Primary Key modifications since the new {} column already exists",
- HOST_COMPONENT_DESIREDSTATE_TABLE, HOST_COMPONENT_DESIREDSTATE_ID_COL);
-
- return;
- }
- // add the new ID column as nullable until we populate
- dbAccessor.addColumn(HOST_COMPONENT_DESIREDSTATE_TABLE,
- new DBColumnInfo(HOST_COMPONENT_DESIREDSTATE_ID_COL, Long.class, null, null, true));
-
- // insert sequence values
- AtomicLong id = new AtomicLong(1);
- Statement statement = null;
- ResultSet resultSet = null;
-
- try {
- statement = dbAccessor.getConnection().createStatement();
-
- if (statement != null) {
- // Select records by old PK
- String selectSQL = String.format(
- "SELECT cluster_id, component_name, host_id, service_name FROM %s", HOST_COMPONENT_DESIREDSTATE_TABLE);
-
- resultSet = statement.executeQuery(selectSQL);
-
- while (resultSet.next()) {
- final Long clusterId = resultSet.getLong("cluster_id");
- final String componentName = resultSet.getString("component_name");
- final Long hostId = resultSet.getLong("host_id");
- final String serviceName = resultSet.getString("service_name");
-
- String updateSQL = String.format(
- "UPDATE %s SET %s = %s WHERE cluster_id = %d AND component_name = '%s' AND service_name = '%s' AND host_id = %d",
- HOST_COMPONENT_DESIREDSTATE_TABLE, HOST_COMPONENT_DESIREDSTATE_ID_COL, id.getAndIncrement(),
- clusterId, componentName, serviceName, hostId);
-
- dbAccessor.executeQuery(updateSQL);
- }
-
- // Add sequence for hostcomponentdesiredstate table ids
- addSequence("hostcomponentdesiredstate_id_seq", id.get(), false);
- }
-
- }
- finally {
- JdbcUtils.closeResultSet(resultSet);
- JdbcUtils.closeStatement(statement);
- }
-
- // make the ID column NON NULL now
- dbAccessor.alterColumn(HOST_COMPONENT_DESIREDSTATE_TABLE,
- new DBColumnInfo(HOST_COMPONENT_DESIREDSTATE_ID_COL, Long.class, null, null, false));
-
- // drop existing PK and create new one on ID column
- String primaryKeyConstraintName = null;
- Configuration.DatabaseType databaseType = configuration.getDatabaseType();
-
- switch (databaseType) {
- case POSTGRES:
- case MYSQL:
- case ORACLE:
- case SQL_SERVER:
- primaryKeyConstraintName = dbAccessor.getPrimaryKeyConstraintName(HOST_COMPONENT_DESIREDSTATE_TABLE);
- break;
-
- default:
- throw new UnsupportedOperationException(String.format("Invalid database type '%s'", databaseType));
-
- }
-
- // warn if we can't find it
- if (null == primaryKeyConstraintName) {
- LOG.warn("Unable to determine the primary key constraint name for {}", HOST_COMPONENT_DESIREDSTATE_TABLE);
- }
- else {
- dbAccessor.dropPKConstraint(HOST_COMPONENT_DESIREDSTATE_TABLE, primaryKeyConstraintName, true);
- }
-
- // create a new PK, matching the name of the constraint found in the SQL files
- dbAccessor.addPKConstraint(HOST_COMPONENT_DESIREDSTATE_TABLE, "PK_hostcomponentdesiredstate", "id");
-
- // create index, ensuring column order matches that of the SQL files
- dbAccessor.addUniqueConstraint(HOST_COMPONENT_DESIREDSTATE_TABLE, HOST_COMPONENT_DESIREDSTATE_INDEX,
- "component_name", "service_name", "host_id", "cluster_id");
- }
-
- protected void updateAtlasConfigs() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
- if (clusters != null) {
- Map<String, Cluster> clusterMap = clusters.getClusters();
- if (clusterMap != null && !clusterMap.isEmpty()) {
- for (final Cluster cluster : clusterMap.values()) {
- updateAtlasHookConfig(cluster, "HIVE", "hive-env", "hive.atlas.hook");
- updateAtlasHookConfig(cluster, "STORM", "storm-env", "storm.atlas.hook");
- updateAtlasHookConfig(cluster, "FALCON", "falcon-env", "falcon.atlas.hook");
- updateAtlasHookConfig(cluster, "SQOOP", "sqoop-env", "sqoop.atlas.hook");
- }
- }
- }
- }
-
- protected void updateAtlasHookConfig(Cluster cluster, String serviceName, String configType, String propertyName) throws AmbariException {
- Set<String> installedServices = cluster.getServices().keySet();
- if (installedServices.contains("ATLAS") && installedServices.contains(serviceName)) {
- Config configEnv = cluster.getDesiredConfigByType(configType);
- if (configEnv != null) {
- Map<String, String> newProperties = new HashMap<>();
- newProperties.put(propertyName, "true");
- boolean updateProperty = configEnv.getProperties().containsKey(propertyName);
- updateConfigurationPropertiesForCluster(cluster, configType, newProperties, updateProperty, true);
- }
- }
- }
-
- /**
- * Updates Hive Interactive's config in hive-interactive-site.
- *
- * @throws AmbariException
- */
- private static final String HIVE_INTERACTIVE_ENV = "hive-interactive-env";
- private static final String HIVE_ENV = "hive-env";
- protected void updateHIVEInteractiveConfigs() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
- if (clusters != null) {
- Map<String, Cluster> clusterMap = clusters.getClusters();
-
- if (clusterMap != null && !clusterMap.isEmpty()) {
- for (final Cluster cluster : clusterMap.values()) {
- Config hiveInteractiveEnv = cluster.getDesiredConfigByType(HIVE_INTERACTIVE_ENV);
- Config hiveEnv = cluster.getDesiredConfigByType(HIVE_ENV);
- if (hiveInteractiveEnv != null) {
- String hsiHeapSize = "512";
- if (hiveEnv != null) {
- if (hiveEnv.getProperties().containsKey("hive.heapsize")) {
- hsiHeapSize = hiveEnv.getProperties().get("hive.heapsize");
- LOG.info("Updating HSI heap size to: " + hsiHeapSize);
- }
- }
- updateConfigurationProperties(HIVE_INTERACTIVE_ENV, Collections.singletonMap("hive_heapsize",
- hsiHeapSize), true, true);
- }
- }
- }
- }
- }
-
- /**
- * Updates Log Search configs.
- *
- * @throws AmbariException
- */
- protected void updateLogSearchConfigs() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
- if (clusters != null) {
- Map<String, Cluster> clusterMap = clusters.getClusters();
-
- if (clusterMap != null && !clusterMap.isEmpty()) {
- for (final Cluster cluster : clusterMap.values()) {
- Config logSearchProperties = cluster.getDesiredConfigByType("logsearch-properties");
- if (logSearchProperties != null) {
- Map<String, String> newProperties = new HashMap<>();
- if (!logSearchProperties.getProperties().containsKey("logsearch.auth.external_auth.enabled"))
- newProperties.put("logsearch.auth.external_auth.enabled", logSearchProperties.getProperties().get("logsearch.external.auth.enabled"));
- if (!logSearchProperties.getProperties().containsKey("logsearch.auth.external_auth.host_url"))
- newProperties.put("logsearch.auth.external_auth.host_url", logSearchProperties.getProperties().get("logsearch.external.auth.host_url"));
- if (!logSearchProperties.getProperties().containsKey("logsearch.auth.external_auth.login_url"))
- newProperties.put("logsearch.auth.external_auth.login_url", logSearchProperties.getProperties().get("logsearch.external.auth.login_url"));
-
- Set<String> removeProperties = new HashSet<>();
- removeProperties.add("logsearch.external.auth.enabled");
- removeProperties.add("logsearch.external.auth.host_url");
- removeProperties.add("logsearch.external.auth.login_url");
-
- updateConfigurationPropertiesForCluster(cluster, "logsearch-properties", newProperties, removeProperties, true, true);
- }
-
- Config logfeederEnvProperties = cluster.getDesiredConfigByType("logfeeder-env");
- if (logfeederEnvProperties != null) {
- String content = logfeederEnvProperties.getProperties().get("content");
- if (content.contains("infra_solr_ssl_enabled")) {
- content = content.replace("infra_solr_ssl_enabled", "logfeeder_use_ssl");
- updateConfigurationPropertiesForCluster(cluster, "logfeeder-env", Collections.singletonMap("content", content), true, true);
- }
- }
-
- Config logsearchEnvProperties = cluster.getDesiredConfigByType("logsearch-env");
- if (logsearchEnvProperties != null) {
- Map<String, String> newProperties = new HashMap<>();
- String content = logsearchEnvProperties.getProperties().get("content");
- if (content.contains("infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'")) {
- content = content.replace("infra_solr_ssl_enabled or logsearch_ui_protocol == 'https'", "logsearch_use_ssl");
- }
- if (!content.equals(logsearchEnvProperties.getProperties().get("content"))) {
- newProperties.put("content", content);
- }
-
- if ("http".equals(logsearchEnvProperties.getProperties().get("logsearch_ui_protocol")) &&
- "/etc/security/serverKeys/logsearch.trustStore.jks".equals(logsearchEnvProperties.getProperties().get("logsearch_truststore_location")) &&
- "/etc/security/serverKeys/logsearch.keyStore.jks".equals(logsearchEnvProperties.getProperties().get("logsearch_keystore_location"))) {
- newProperties.put("logsearch_truststore_location", "/etc/ambari-logsearch-portal/conf/keys/logsearch.jks");
- newProperties.put("logsearch_keystore_location", "/etc/ambari-logsearch-portal/conf/keys/logsearch.jks");
- }
-
- Set<String> removeProperties = new HashSet<>();
- removeProperties.add("logsearch_solr_audit_logs_use_ranger");
- removeProperties.add("logsearch_solr_audit_logs_zk_node");
- removeProperties.add("logsearch_solr_audit_logs_zk_quorum");
-
- updateConfigurationPropertiesForCluster(cluster, "logsearch-env", newProperties, removeProperties, true, true);
- }
-
- Config logfeederLog4jProperties = cluster.getDesiredConfigByType("logfeeder-log4j");
- if (logfeederLog4jProperties != null) {
- Map<String, String> newProperties = new HashMap<>();
-
- String content = logfeederLog4jProperties.getProperties().get("content");
- content = SchemaUpgradeUtil.extractProperty(content, "logfeeder_log_maxfilesize", "logfeeder_log_maxfilesize",
- " <param name=\"file\" value=\"\\{\\{logfeeder_log_dir}}/logfeeder.log\"/>\n" +
- " <param name=\"append\" value=\"true\"/>\n" +
- " <param name=\"maxFileSize\" value=\"(\\w+)MB\"/>", "10", newProperties);
- content = SchemaUpgradeUtil.extractProperty(content, "logfeeder_log_maxbackupindex", "logfeeder_log_maxbackupindex",
- " <param name=\"file\" value=\"\\{\\{logfeeder_log_dir}}/logfeeder.log\"/>\n" +
- " <param name=\"append\" value=\"true\"/>\n" +
- " <param name=\"maxFileSize\" value=\"\\{\\{logfeeder_log_maxfilesize}}MB\"/>\n" +
- " <param name=\"maxBackupIndex\" value=\"(\\w+)\"/>", "10", newProperties);
- content = SchemaUpgradeUtil.extractProperty(content, "logfeeder_json_log_maxfilesize", "logfeeder_json_log_maxfilesize",
- " <param name=\"file\" value=\"\\{\\{logfeeder_log_dir}}/logsearch-logfeeder.json\" />\n" +
- " <param name=\"append\" value=\"true\" />\n" +
- " <param name=\"maxFileSize\" value=\"(\\w+)MB\" />", "10", newProperties);
- content = SchemaUpgradeUtil.extractProperty(content, "logfeeder_json_log_maxbackupindex", "logfeeder_json_log_maxbackupindex",
- " <param name=\"file\" value=\"\\{\\{logfeeder_log_dir}}/logsearch-logfeeder.json\" />\n" +
- " <param name=\"append\" value=\"true\" />\n" +
- " <param name=\"maxFileSize\" value=\"\\{\\{logfeeder_json_log_maxfilesize}}MB\" />\n" +
- " <param name=\"maxBackupIndex\" value=\"(\\w+)\" />", "10", newProperties);
-
- newProperties.put("content", content);
- updateConfigurationPropertiesForCluster(cluster, "logfeeder-log4j", newProperties, true, true);
- }
-
- Config logsearchLog4jProperties = cluster.getDesiredConfigByType("logsearch-log4j");
- if (logsearchLog4jProperties != null) {
- Map<String, String> newProperties = new HashMap<>();
-
- String content = logsearchLog4jProperties.getProperties().get("content");
- if (content.contains("{{logsearch_log_dir}}/logsearch.err")) {
- content = content.replace("{{logsearch_log_dir}}/logsearch.err", "{{logsearch_log_dir}}/logsearch.log");
- }
- if (content.contains("<priority value=\"warn\"/>")) {
- content = content.replace("<priority value=\"warn\"/>", "<priority value=\"info\"/>");
- }
-
-
- content = SchemaUpgradeUtil.extractProperty(content, "logsearch_log_maxfilesize", "logsearch_log_maxfilesize",
- " <param name=\"file\" value=\"\\{\\{logsearch_log_dir}}/logsearch.log\" />\n" +
- " <param name=\"Threshold\" value=\"info\" />\n" +
- " <param name=\"append\" value=\"true\" />\n" +
- " <param name=\"maxFileSize\" value=\"(\\w+)MB\" />\n", "10", newProperties);
- content = SchemaUpgradeUtil.extractProperty(content, "logsearch_log_maxbackupindex", "logsearch_log_maxbackupindex",
- " <param name=\"file\" value=\"\\{\\{logsearch_log_dir}}/logsearch.log\" />\n" +
- " <param name=\"Threshold\" value=\"info\" />\n" +
- " <param name=\"append\" value=\"true\" />\n" +
- " <param name=\"maxFileSize\" value=\"\\{\\{logsearch_log_maxfilesize}}MB\" />\n" +
- " <param name=\"maxBackupIndex\" value=\"(\\w+)\" />\n", "10", newProperties);
- content = SchemaUpgradeUtil.extractProperty(content, "logsearch_json_log_maxfilesize", "logsearch_json_log_maxfilesize",
- " <param name=\"file\" value=\"\\{\\{logsearch_log_dir}}/logsearch.json\"/>\n" +
- " <param name=\"append\" value=\"true\"/>\n" +
- " <param name=\"maxFileSize\" value=\"(\\w+)MB\"/>\n", "10", newProperties);
- content = SchemaUpgradeUtil.extractProperty(content, "logsearch_json_log_maxbackupindex", "logsearch_json_log_maxbackupindex",
- " <param name=\"file\" value=\"\\{\\{logsearch_log_dir}}/logsearch.json\"/>\n" +
- " <param name=\"append\" value=\"true\"/>\n" +
- " <param name=\"maxFileSize\" value=\"\\{\\{logsearch_json_log_maxfilesize}}MB\"/>\n" +
- " <param name=\"maxBackupIndex\" value=\"(\\w+)\"/>\n", "10", newProperties);
- content = SchemaUpgradeUtil.extractProperty(content, "logsearch_audit_log_maxfilesize", "logsearch_audit_log_maxfilesize",
- " <param name=\"file\" value=\"\\{\\{logsearch_log_dir}}/logsearch-audit.json\"/>\n" +
- " <param name=\"append\" value=\"true\"/>\n" +
- " <param name=\"maxFileSize\" value=\"(\\w+)MB\"/>\n", "10", newProperties);
- content = SchemaUpgradeUtil.extractProperty(content, "logsearch_audit_log_maxbackupindex", "logsearch_audit_log_maxbackupindex",
- " <param name=\"file\" value=\"\\{\\{logsearch_log_dir}}/logsearch-audit.json\"/>\n" +
- " <param name=\"append\" value=\"true\"/>\n" +
- " <param name=\"maxFileSize\" value=\"\\{\\{logsearch_audit_log_maxfilesize}}MB\"/>\n" +
- " <param name=\"maxBackupIndex\" value=\"(\\w+)\"/>\n", "10", newProperties);
- content = SchemaUpgradeUtil.extractProperty(content, "logsearch_perf_log_maxfilesize", "logsearch_perf_log_maxfilesize",
- " <param name=\"file\" value=\"\\{\\{logsearch_log_dir}}/logsearch-performance.json\"/>\n" +
- " <param name=\"Threshold\" value=\"info\"/>\n" +
- " <param name=\"append\" value=\"true\"/>\n" +
- " <param name=\"maxFileSize\" value=\"(\\w+)MB\"/>\n", "10", newProperties);
- content = SchemaUpgradeUtil.extractProperty(content, "logsearch_perf_log_maxbackupindex", "logsearch_perf_log_maxbackupindex",
- " <param name=\"file\" value=\"\\{\\{logsearch_log_dir}}/logsearch-performance.json\"/>\n" +
- " <param name=\"Threshold\" value=\"info\"/>\n" +
- " <param name=\"append\" value=\"true\"/>\n" +
- " <param name=\"maxFileSize\" value=\"\\{\\{logsearch_perf_log_maxfilesize}}MB\"/>\n" +
- " <param name=\"maxBackupIndex\" value=\"(\\w+)\"/>\n", "10", newProperties);
-
- newProperties.put("content", content);
- if (!content.equals(logsearchLog4jProperties.getProperties().get("content"))) {
- updateConfigurationPropertiesForCluster(cluster, "logsearch-log4j", newProperties, true, true);
- }
- }
- }
- }
- }
- }
-
- /**
- * Updates Ambari Infra configs.
- *
- * @throws AmbariException
- */
- protected void updateAmbariInfraConfigs() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
- if (clusters != null) {
- Map<String, Cluster> clusterMap = clusters.getClusters();
-
- if (clusterMap != null && !clusterMap.isEmpty()) {
- for (final Cluster cluster : clusterMap.values()) {
- Config infraSolrEnvProperties = cluster.getDesiredConfigByType("infra-solr-env");
- if (infraSolrEnvProperties != null) {
- String content = infraSolrEnvProperties.getProperties().get("content");
- if (content.contains("SOLR_SSL_TRUST_STORE={{infra_solr_keystore_location}}")) {
- content = content.replace("SOLR_SSL_TRUST_STORE={{infra_solr_keystore_location}}", "SOLR_SSL_TRUST_STORE={{infra_solr_truststore_location}}");
- }
- if (content.contains("SOLR_SSL_TRUST_STORE_PASSWORD={{infra_solr_keystore_password}}")) {
- content = content.replace("SOLR_SSL_TRUST_STORE_PASSWORD={{infra_solr_keystore_password}}", "SOLR_SSL_TRUST_STORE_PASSWORD={{infra_solr_truststore_password}}");
- }
- if (content.contains("SOLR_KERB_NAME_RULES={{infra_solr_kerberos_name_rules}}")) {
- content = content.replace("SOLR_KERB_NAME_RULES={{infra_solr_kerberos_name_rules}}", "SOLR_KERB_NAME_RULES=\"{{infra_solr_kerberos_name_rules}}\"");
- }
- if (content.contains(" -Dsolr.kerberos.name.rules=${SOLR_KERB_NAME_RULES}")) {
- content = content.replace(" -Dsolr.kerberos.name.rules=${SOLR_KERB_NAME_RULES}", "");
- }
- if (!content.equals(infraSolrEnvProperties.getProperties().get("content"))) {
- updateConfigurationPropertiesForCluster(cluster, "infra-solr-env", Collections.singletonMap("content", content), true, true);
- }
- }
-
- Config infraSolrLog4jProperties = cluster.getDesiredConfigByType("infra-solr-log4j");
- if (infraSolrLog4jProperties != null) {
- Map<String, String> newProperties = new HashMap<>();
-
- String content = infraSolrLog4jProperties.getProperties().get("content");
- content = SchemaUpgradeUtil.extractProperty(content, "infra_log_maxfilesize", "infra_log_maxfilesize",
- "log4j.appender.file.MaxFileSize=(\\w+)MB", "10", newProperties);
- content = SchemaUpgradeUtil.extractProperty(content, "infra_log_maxbackupindex", "infra_log_maxbackupindex",
- "log4j.appender.file.MaxBackupIndex=(\\w+)\n", "9", newProperties);
-
- newProperties.put("content", content);
- updateConfigurationPropertiesForCluster(cluster, "infra-solr-log4j", newProperties, true, true);
- }
-
- Config infraSolrClientLog4jProperties = cluster.getDesiredConfigByType("infra-solr-client-log4j");
- if (infraSolrClientLog4jProperties != null) {
- Map<String, String> newProperties = new HashMap<>();
-
- String content = infraSolrClientLog4jProperties.getProperties().get("content");
- if (content.contains("infra_client_log")) {
- content = content.replace("infra_client_log", "solr_client_log");
- }
-
- content = SchemaUpgradeUtil.extractProperty(content, "infra_client_log_maxfilesize", "solr_client_log_maxfilesize",
- "log4j.appender.file.MaxFileSize=(\\w+)MB", "80", newProperties);
- content = SchemaUpgradeUtil.extractProperty(content, "infra_client_log_maxbackupindex", "solr_client_log_maxbackupindex",
- "log4j.appender.file.MaxBackupIndex=(\\w+)\n", "60", newProperties);
-
- newProperties.put("content", content);
- updateConfigurationPropertiesForCluster(cluster, "infra-solr-client-log4j", newProperties, true, true);
- }
- }
- }
- }
- }
-
- /**
- * Add permissions for managing service auto-start.
- * <p>
- * <ul>
- * <li>SERVICE.MANAGE_AUTO_START permissions for SERVICE.ADMINISTRATOR, CLUSTER.OPERATOR, CLUSTER.ADMINISTRATOR, AMBARI.ADMINISTRATOR</li>
- * <li>CLUSTER.MANAGE_AUTO_START permissions for CLUSTER.OPERATOR, CLUSTER.ADMINISTRATOR, AMBARI.ADMINISTRATOR</li>
- * </ul>
- */
- protected void addManageServiceAutoStartPermissions() throws SQLException {
- Collection<String> roles;
-
- // Add service-level auto-start permission
- roles = Arrays.asList(
- "AMBARI.ADMINISTRATOR:AMBARI",
- "CLUSTER.ADMINISTRATOR:CLUSTER",
- "CLUSTER.OPERATOR:CLUSTER",
- "SERVICE.ADMINISTRATOR:CLUSTER");
- addRoleAuthorization("SERVICE.MANAGE_AUTO_START", "Manage service auto-start", roles);
-
- // Add cluster-level auto start-permission
- roles = Arrays.asList(
- "AMBARI.ADMINISTRATOR:AMBARI",
- "CLUSTER.ADMINISTRATOR:CLUSTER",
- "CLUSTER.OPERATOR:CLUSTER");
- addRoleAuthorization("CLUSTER.MANAGE_AUTO_START", "Manage service auto-start configuration", roles);
- }
-
- /**
- * Add permissions for managing alert notifications configuration.
- * <p>
- * <ul>
- * <li>CLUSTER.MANAGE_ALERT_NOTIFICATIONS permissions for AMBARI.ADMINISTRATOR, CLUSTER.ADMINISTRATOR</li>
- * </ul>
- */
- protected void addManageAlertNotificationsPermissions() throws SQLException {
- Collection<String> roles;
- roles = Arrays.asList(
- "AMBARI.ADMINISTRATOR:AMBARI",
- "CLUSTER.ADMINISTRATOR:CLUSTER");
- addRoleAuthorization("CLUSTER.MANAGE_ALERT_NOTIFICATIONS", "Manage alert notifications configuration", roles);
- }
-
- /**
- * Updates Ranger admin url for Ranger plugin supported configs.
- *
- * @throws AmbariException
- */
- protected void updateRangerUrlConfigs() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) {
-
- Config ranger_admin_properties = cluster.getDesiredConfigByType("admin-properties");
- if(null != ranger_admin_properties) {
- String policyUrl = ranger_admin_properties.getProperties().get("policymgr_external_url");
- if (null != policyUrl) {
- updateRangerUrl(cluster, "ranger-hdfs-security", "ranger.plugin.hdfs.policy.rest.url", policyUrl);
- updateRangerUrl(cluster, "ranger-hive-security", "ranger.plugin.hive.policy.rest.url", policyUrl);
- updateRangerUrl(cluster, "ranger-hbase-security", "ranger.plugin.hbase.policy.rest.url", policyUrl);
- updateRangerUrl(cluster, "ranger-knox-security", "ranger.plugin.knox.policy.rest.url", policyUrl);
- updateRangerUrl(cluster, "ranger-storm-security", "ranger.plugin.storm.policy.rest.url", policyUrl);
- updateRangerUrl(cluster, "ranger-yarn-security", "ranger.plugin.yarn.policy.rest.url", policyUrl);
- updateRangerUrl(cluster, "ranger-kafka-security", "ranger.plugin.kafka.policy.rest.url", policyUrl);
- updateRangerUrl(cluster, "ranger-atlas-security", "ranger.plugin.atlas.policy.rest.url", policyUrl);
- updateRangerUrl(cluster, "ranger-kms-security", "ranger.plugin.kms.policy.rest.url", policyUrl);
- }
- }
- }
- }
-
- protected void updateRangerUrl(Cluster cluster, String configType, String configProperty, String policyUrl) throws AmbariException {
- Config componentSecurity = cluster.getDesiredConfigByType(configType);
- if(componentSecurity != null && componentSecurity.getProperties().containsKey(configProperty)) {
- Map<String, String> updateProperty = new HashMap<>();
- updateProperty.put(configProperty, policyUrl);
- updateConfigurationPropertiesForCluster(cluster, configType, updateProperty, true, false);
- }
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/af1bf85c/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog300.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog300.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog300.java
index 9b44c53..5fdc885 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog300.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog300.java
@@ -76,6 +76,7 @@ public class UpgradeCatalog300 extends AbstractUpgradeCatalog {
protected static final String CLUSTER_CONFIG_MAPPING_TABLE = "clusterconfigmapping";
protected static final String HOST_ROLE_COMMAND_TABLE = "host_role_command";
protected static final String HRC_OPS_DISPLAY_NAME_COLUMN = "ops_display_name";
+ protected static final String COMPONENT_TABLE = "servicecomponentdesiredstate";
@Inject
DaoUtils daoUtils;
@@ -112,7 +113,7 @@ public class UpgradeCatalog300 extends AbstractUpgradeCatalog {
*/
@Override
public String getSourceVersion() {
- return "2.5.0";
+ return "2.5.2";
}
/**
@@ -181,7 +182,7 @@ public class UpgradeCatalog300 extends AbstractUpgradeCatalog {
* @throws SQLException
*/
protected void addServiceComponentColumn() throws SQLException {
- dbAccessor.addColumn(UpgradeCatalog250.COMPONENT_TABLE,
+ dbAccessor.addColumn(COMPONENT_TABLE,
new DBColumnInfo("repo_state", String.class, 255,
RepositoryVersionState.NOT_REQUIRED.name(), false));
}