You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2017/10/11 23:03:44 UTC
[17/50] [abbrv] ambari git commit: Merge remote-tracking branch
'origin/trunk' into branch-feature-AMBARI-14714
Merge remote-tracking branch 'origin/trunk' into branch-feature-AMBARI-14714
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c36afcdd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c36afcdd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c36afcdd
Branch: refs/heads/branch-feature-AMBARI-14714-ui
Commit: c36afcdd051fa28cff2d299f1b97ed92e32fbbe5
Parents: ba1ec6d e61556c
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Sun Oct 8 23:00:20 2017 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Sun Oct 8 23:00:20 2017 -0700
----------------------------------------------------------------------
ambari-agent/pom.xml | 4 +-
.../main/python/ambari_agent/AmbariConfig.py | 74 +-
.../ambari_agent/CustomServiceOrchestrator.py | 6 +-
.../src/main/python/ambari_agent/FileCache.py | 5 +-
.../main/python/ambari_agent/ProcessHelper.py | 71 --
.../src/main/python/ambari_agent/StatusCheck.py | 142 ----
.../src/main/python/ambari_agent/main.py | 12 +-
.../test/python/ambari_agent/TestFileCache.py | 4 +-
.../test/python/ambari_agent/TestLiveStatus.py | 5 +-
.../src/test/python/ambari_agent/TestMain.py | 13 +-
.../python/ambari_agent/TestProcessHelper.py | 70 --
.../test/python/ambari_agent/TestSecurity.py | 1 -
.../test/python/ambari_agent/TestStatusCheck.py | 180 -----
.../resource_management/TestPackageResource.py | 6 +-
.../python/resource_management/TestScript.py | 26 +-
.../core/providers/package/__init__.py | 4 +-
.../core/providers/package/apt.py | 35 +-
.../core/providers/package/choco.py | 8 +-
.../core/providers/package/yumrpm.py | 10 +-
.../core/providers/package/zypper.py | 7 +-
.../core/resources/packaging.py | 14 +-
.../libraries/functions/component_version.py | 26 +-
.../libraries/functions/conf_select.py | 362 +++------
.../libraries/functions/repository_util.py | 8 +-
.../libraries/functions/stack_select.py | 69 +-
.../libraries/script/script.py | 22 +-
ambari-infra/ambari-infra-assembly/pom.xml | 8 +
.../src/main/package/deb/manager/postinst | 5 +
.../src/main/package/deb/solr-client/postinst | 13 +
.../src/main/package/rpm/manager/postinstall.sh | 20 +
.../main/package/rpm/solr-client/postinstall.sh | 28 +
.../org/apache/ambari/infra/InfraManager.java | 4 +-
.../conf/batch/InfraManagerBatchConfig.java | 86 --
.../infra/job/dummy/DummyJobConfiguration.java | 108 +++
.../src/main/resources/infraManager.sh | 10 +-
ambari-infra/ambari-infra-solr-client/build.xml | 3 +
.../src/main/python/solrDataManager.py | 150 ++--
.../src/main/resources/solrIndexHelper.sh | 5 +-
.../logsearch/steps/LogSearchUISteps.java | 4 +-
.../resources/stories/selenium/login.ui.story | 4 +-
.../ambari-logsearch-web/package.json | 17 +-
ambari-logsearch/ambari-logsearch-web/pom.xml | 6 +-
.../ambari-logsearch-web/src/app/app.module.ts | 6 +
.../src/app/classes/active-service-log-entry.ts | 23 +
.../src/app/classes/histogram-options.ts | 36 +
.../src/app/classes/list-item.class.ts | 25 -
.../src/app/classes/list-item.ts | 26 +
.../src/app/classes/models/app-settings.ts | 27 +
.../src/app/classes/models/app-state.ts | 43 +
.../src/app/classes/models/audit-log-field.ts | 225 ++++++
.../src/app/classes/models/audit-log.ts | 46 ++
.../src/app/classes/models/bar-graph.ts | 24 +
.../src/app/classes/models/common-entry.ts | 22 +
.../src/app/classes/models/count.ts | 22 +
.../src/app/classes/models/filter.ts | 25 +
.../src/app/classes/models/graph.ts | 23 +
.../src/app/classes/models/log-field.ts | 27 +
.../src/app/classes/models/log.ts | 38 +
.../src/app/classes/models/node.ts | 30 +
.../src/app/classes/models/service-log-field.ts | 107 +++
.../src/app/classes/models/service-log.ts | 27 +
.../app/classes/models/solr-collection-state.ts | 23 +
.../src/app/classes/models/store.ts | 180 +++++
.../src/app/classes/models/user-config.ts | 26 +
.../queries/audit-logs-query-params.class.ts | 46 --
.../classes/queries/audit-logs-query-params.ts | 46 ++
.../app/classes/queries/query-params.class.ts | 23 -
.../src/app/classes/queries/query-params.ts | 23 +
...ce-logs-histogram-query-params.class.spec.ts | 203 -----
...service-logs-histogram-query-params.class.ts | 70 --
.../service-logs-histogram-query-params.spec.ts | 203 +++++
.../service-logs-histogram-query-params.ts | 70 ++
.../queries/service-logs-query-params.class.ts | 30 -
.../queries/service-logs-query-params.ts | 30 +
.../service-logs-truncated-query-params.ts | 36 +
.../app/classes/service-log-context-entry.ts | 26 +
.../dropdown-button.component.html | 3 +-
.../dropdown-button.component.spec.ts | 5 +-
.../dropdown-button.component.ts | 2 +-
.../dropdown-list/dropdown-list.component.html | 2 +-
.../dropdown-list.component.spec.ts | 12 +-
.../dropdown-list/dropdown-list.component.ts | 11 +-
.../filter-button.component.spec.ts | 5 +-
.../filter-button/filter-button.component.ts | 2 +-
.../filter-dropdown.component.spec.ts | 5 +-
.../filters-panel/filters-panel.component.html | 3 +-
.../filters-panel/filters-panel.component.less | 2 +-
.../filters-panel.component.spec.ts | 5 +-
.../filters-panel/filters-panel.component.ts | 4 +-
.../log-context/log-context.component.html | 33 +
.../log-context/log-context.component.less | 23 +
.../log-context/log-context.component.spec.ts | 108 +++
.../log-context/log-context.component.ts | 91 +++
.../log-file-entry.component.html | 20 +
.../log-file-entry.component.less | 31 +
.../log-file-entry.component.spec.ts | 56 ++
.../log-file-entry/log-file-entry.component.ts | 51 ++
.../logs-container.component.html | 12 +-
.../logs-container.component.spec.ts | 5 +-
.../logs-container/logs-container.component.ts | 44 +-
.../logs-list/logs-list.component.html | 30 +-
.../logs-list/logs-list.component.less | 36 +-
.../logs-list/logs-list.component.spec.ts | 3 +
.../components/logs-list/logs-list.component.ts | 34 +-
.../main-container.component.html | 7 +
.../main-container.component.less | 4 +
.../main-container.component.spec.ts | 13 +-
.../main-container/main-container.component.ts | 32 +-
.../menu-button/menu-button.component.spec.ts | 5 +-
.../menu-button/menu-button.component.ts | 2 +-
.../search-box/search-box.component.ts | 2 +-
.../time-histogram.component.less | 22 +-
.../time-histogram/time-histogram.component.ts | 94 ++-
.../timezone-picker.component.spec.ts | 5 +-
.../src/app/components/variables.less | 30 +
.../src/app/models/app-settings.model.ts | 27 -
.../src/app/models/app-state.model.ts | 35 -
.../src/app/models/audit-log-field.model.ts | 225 ------
.../src/app/models/audit-log.model.ts | 46 --
.../src/app/models/bar-graph.model.ts | 24 -
.../src/app/models/common-entry.model.ts | 22 -
.../src/app/models/count.model.ts | 22 -
.../src/app/models/filter.model.ts | 25 -
.../src/app/models/graph.model.ts | 23 -
.../src/app/models/log-field.model.ts | 27 -
.../src/app/models/log.model.ts | 37 -
.../src/app/models/node.model.ts | 29 -
.../src/app/models/service-log-field.model.ts | 107 ---
.../src/app/models/service-log.model.ts | 27 -
.../app/models/solr-collection-state.model.ts | 23 -
.../src/app/models/store.model.ts | 169 ----
.../src/app/models/user-config.model.ts | 26 -
.../services/component-actions.service.spec.ts | 8 +-
.../app/services/component-actions.service.ts | 52 +-
.../component-generator.service.spec.ts | 10 +-
.../app/services/component-generator.service.ts | 23 +-
.../src/app/services/filtering.service.spec.ts | 27 +
.../src/app/services/filtering.service.ts | 97 ++-
.../src/app/services/http-client.service.ts | 15 +-
.../app/services/logs-container.service.spec.ts | 8 +-
.../src/app/services/logs-container.service.ts | 70 +-
.../services/storage/app-settings.service.ts | 4 +-
.../app/services/storage/app-state.service.ts | 4 +-
.../storage/audit-logs-fields.service.ts | 2 +-
.../app/services/storage/audit-logs.service.ts | 2 +-
.../app/services/storage/clusters.service.ts | 2 +-
.../app/services/storage/components.service.ts | 2 +-
.../src/app/services/storage/filters.service.ts | 2 +-
.../src/app/services/storage/graphs.service.ts | 2 +-
.../src/app/services/storage/hosts.service.ts | 2 +-
.../app/services/storage/reducers.service.ts | 2 +
.../storage/service-logs-fields.service.ts | 2 +-
.../service-logs-histogram-data.service.ts | 2 +-
.../storage/service-logs-truncated.service.ts | 32 +
.../services/storage/service-logs.service.ts | 2 +-
.../services/storage/user-configs.service.ts | 2 +-
.../src/assets/i18n/en.json | 7 +-
.../ambari-logsearch-web/webpack.config.js | 21 +-
ambari-logsearch/ambari-logsearch-web/yarn.lock | 288 +++++--
.../timeline/HBaseTimelineMetricStore.java | 8 +-
.../timeline/HBaseTimelineMetricStoreTest.java | 36 +-
ambari-server/pom.xml | 2 +
ambari-server/src/main/assemblies/server.xml | 20 +-
.../actionmanager/ExecutionCommandWrapper.java | 3 +-
.../ambari/server/actionmanager/Stage.java | 3 +-
.../ambari/server/agent/HeartbeatMonitor.java | 3 +-
.../ambari/server/agent/StatusCommand.java | 16 +
.../server/api/services/AmbariMetaInfo.java | 61 +-
.../AmbariManagementControllerImpl.java | 22 +-
.../server/controller/ControllerModule.java | 4 +
.../server/controller/KerberosHelperImpl.java | 16 +-
.../controller/ResourceProviderFactory.java | 8 +
.../server/controller/StackVersionResponse.java | 29 -
.../AbstractControllerResourceProvider.java | 4 +
.../internal/AlertTargetResourceProvider.java | 3 +-
.../internal/ClientConfigResourceProvider.java | 2 -
.../internal/DefaultProviderModule.java | 4 -
.../internal/StackArtifactResourceProvider.java | 35 +-
.../internal/UpgradeResourceProvider.java | 77 +-
.../internal/ViewInstanceResourceProvider.java | 147 ++--
.../upgrade/HostVersionOutOfSyncListener.java | 4 +
.../ambari/server/mpack/MpackGenerator.java | 17 -
.../orm/entities/RepositoryVersionEntity.java | 34 +-
.../server/orm/entities/WidgetLayoutEntity.java | 6 +-
.../upgrades/AbstractUpgradeServerAction.java | 6 +-
.../upgrades/AutoSkipFailedSummaryAction.java | 15 +-
.../upgrades/ComponentVersionCheckAction.java | 2 +-
.../serveraction/upgrades/ConfigureAction.java | 11 +-
.../upgrades/FinalizeUpgradeAction.java | 7 +-
.../FixCapacitySchedulerOrderingPolicy.java | 12 +-
.../serveraction/upgrades/FixLzoCodecPath.java | 10 +-
.../upgrades/FixOozieAdminUsers.java | 10 +-
.../upgrades/FixYarnWebServiceUrl.java | 11 +-
.../upgrades/HBaseConfigCalculation.java | 10 +-
.../HBaseEnvMaxDirectMemorySizeAction.java | 11 +-
.../upgrades/HiveEnvClasspathAction.java | 11 +-
.../upgrades/HiveZKQuorumConfigAction.java | 13 +-
.../upgrades/KerberosKeytabsAction.java | 13 +-
.../upgrades/ManualStageAction.java | 3 +-
.../upgrades/OozieConfigCalculation.java | 11 +-
.../upgrades/PreconfigureKerberosAction.java | 2 +-
.../upgrades/RangerConfigCalculation.java | 11 +-
.../RangerKerberosConfigCalculation.java | 11 +-
.../upgrades/RangerKmsProxyConfig.java | 11 +-
.../RangerUsersyncConfigCalculation.java | 11 +-
.../upgrades/RangerWebAlertConfigAction.java | 9 +-
.../upgrades/SparkShufflePropertyConfig.java | 11 +-
.../upgrades/UpdateDesiredRepositoryAction.java | 10 +-
.../upgrades/UpgradeUserKerberosDescriptor.java | 6 +-
.../upgrades/YarnConfigCalculation.java | 11 +-
.../ambari/server/stack/ServiceDirectory.java | 3 +-
.../ambari/server/stack/StackContext.java | 15 +-
.../ambari/server/stack/StackDirectory.java | 74 +-
.../ambari/server/stack/StackManager.java | 11 +-
.../apache/ambari/server/stack/StackModule.java | 16 -
.../org/apache/ambari/server/state/Host.java | 17 +
.../apache/ambari/server/state/ServiceInfo.java | 2 +-
.../apache/ambari/server/state/StackInfo.java | 39 -
.../server/state/cluster/ClusterImpl.java | 2 +-
.../ambari/server/state/host/HostImpl.java | 47 ++
.../AbstractKerberosDescriptorContainer.java | 26 +-
.../state/kerberos/KerberosDescriptor.java | 3 +-
.../ambari/server/state/stack/UpgradePack.java | 6 +
.../stack/upgrade/ServerSideActionTask.java | 15 +
.../state/stack/upgrade/TaskParameter.java | 41 +
.../svccomphost/ServiceComponentHostImpl.java | 9 +-
.../server/topology/AsyncCallableService.java | 110 ++-
.../ambari/server/topology/TopologyManager.java | 26 +-
.../topology/tasks/ConfigureClusterTask.java | 124 +--
.../server/upgrade/UpgradeCatalog260.java | 31 +
.../ambari_server/dbConfiguration_linux.py | 9 +-
.../python/ambari_server/resourceFilesKeeper.py | 7 +-
.../python/ambari_server/serverConfiguration.py | 2 +-
.../main/python/ambari_server/setupMpacks.py | 2 +-
.../1.6.1.2.2.0/configuration/accumulo-env.xml | 6 +
.../0.1.0/configuration/infra-solr-env.xml | 6 +
.../0.1.0/configuration/ams-env.xml | 6 +
.../AMBARI_METRICS/0.1.0/service_advisor.py | 787 +++++++++++++++++++
.../ATLAS/0.1.0.2.3/configuration/atlas-env.xml | 6 +
.../ATLAS/0.7.0.3.0/configuration/atlas-env.xml | 6 +
.../ATLAS/0.7.0.3.0/service_advisor.py | 5 +-
.../DRUID/0.10.1/configuration/druid-broker.xml | 106 +++
.../DRUID/0.10.1/configuration/druid-common.xml | 270 +++++++
.../0.10.1/configuration/druid-coordinator.xml | 43 +
.../DRUID/0.10.1/configuration/druid-env.xml | 248 ++++++
.../0.10.1/configuration/druid-historical.xml | 94 +++
.../DRUID/0.10.1/configuration/druid-log4j.xml | 84 ++
.../0.10.1/configuration/druid-logrotate.xml | 68 ++
.../configuration/druid-middlemanager.xml | 122 +++
.../0.10.1/configuration/druid-overlord.xml | 52 ++
.../DRUID/0.10.1/configuration/druid-router.xml | 59 ++
.../common-services/DRUID/0.10.1/metainfo.xml | 223 ++++++
.../DRUID/0.10.1/package/scripts/broker.py | 28 +
.../DRUID/0.10.1/package/scripts/coordinator.py | 28 +
.../DRUID/0.10.1/package/scripts/druid.py | 307 ++++++++
.../DRUID/0.10.1/package/scripts/druid_node.py | 114 +++
.../DRUID/0.10.1/package/scripts/historical.py | 28 +
.../0.10.1/package/scripts/middlemanager.py | 28 +
.../DRUID/0.10.1/package/scripts/overlord.py | 28 +
.../DRUID/0.10.1/package/scripts/params.py | 200 +++++
.../DRUID/0.10.1/package/scripts/router.py | 28 +
.../0.10.1/package/scripts/service_check.py | 44 ++
.../0.10.1/package/scripts/status_params.py | 24 +
.../DRUID/0.10.1/quicklinks/quicklinks.json | 37 +
.../DRUID/0.10.1/role_command_order.json | 17 +
.../DRUID/0.10.1/themes/theme.json | 120 +++
.../DRUID/0.9.2/configuration/druid-broker.xml | 100 ---
.../DRUID/0.9.2/configuration/druid-common.xml | 270 -------
.../0.9.2/configuration/druid-coordinator.xml | 43 -
.../DRUID/0.9.2/configuration/druid-env.xml | 242 ------
.../0.9.2/configuration/druid-historical.xml | 88 ---
.../DRUID/0.9.2/configuration/druid-log4j.xml | 84 --
.../0.9.2/configuration/druid-logrotate.xml | 68 --
.../0.9.2/configuration/druid-middlemanager.xml | 122 ---
.../0.9.2/configuration/druid-overlord.xml | 52 --
.../DRUID/0.9.2/configuration/druid-router.xml | 59 --
.../common-services/DRUID/0.9.2/metainfo.xml | 223 ------
.../DRUID/0.9.2/package/scripts/broker.py | 28 -
.../DRUID/0.9.2/package/scripts/coordinator.py | 28 -
.../DRUID/0.9.2/package/scripts/druid.py | 307 --------
.../DRUID/0.9.2/package/scripts/druid_node.py | 114 ---
.../DRUID/0.9.2/package/scripts/historical.py | 28 -
.../0.9.2/package/scripts/middlemanager.py | 28 -
.../DRUID/0.9.2/package/scripts/overlord.py | 28 -
.../DRUID/0.9.2/package/scripts/params.py | 200 -----
.../DRUID/0.9.2/package/scripts/router.py | 28 -
.../0.9.2/package/scripts/service_check.py | 44 --
.../0.9.2/package/scripts/status_params.py | 24 -
.../DRUID/0.9.2/quicklinks/quicklinks.json | 37 -
.../DRUID/0.9.2/role_command_order.json | 17 -
.../DRUID/0.9.2/themes/theme.json | 120 ---
.../0.5.0.2.1/configuration/falcon-env.xml | 6 +
.../FLUME/1.4.0.2.0/configuration/flume-env.xml | 6 +
.../0.96.0.2.0/configuration/hbase-env.xml | 6 +
.../HBASE/2.0.0.3.0/configuration/hbase-env.xml | 6 +
.../HDFS/2.1.0.2.0/configuration/hadoop-env.xml | 10 +
.../HDFS/2.1.0.2.0/configuration/hdfs-site.xml | 1 +
.../common-services/HDFS/2.1.0.2.0/widgets.json | 4 +-
.../HDFS/3.0.0.3.0/configuration/hadoop-env.xml | 10 +
.../HDFS/3.0.0.3.0/configuration/hdfs-site.xml | 1 +
.../HIVE/0.12.0.2.0/configuration/hive-env.xml | 12 +
.../package/scripts/hive_server_upgrade.py | 5 -
.../0.12.0.2.0/package/scripts/params_linux.py | 5 +-
.../0.12.0.2.0/package/scripts/status_params.py | 8 +-
.../HIVE/0.12.0.2.0/package/scripts/webhcat.py | 2 +-
.../HIVE/2.1.0.3.0/configuration/hive-env.xml | 12 +
.../0.10.0.3.0/configuration/kafka-env.xml | 6 +
.../KAFKA/0.8.1/configuration/kafka-env.xml | 6 +
.../KNOX/0.5.0.2.2/configuration/knox-env.xml | 10 +
.../KNOX/0.5.0.3.0/configuration/knox-env.xml | 10 +
.../0.5.0/configuration/logsearch-env.xml | 6 +
.../1.0.0.2.3/configuration/mahout-env.xml | 6 +
.../OOZIE/4.0.0.2.0/configuration/oozie-env.xml | 10 +
.../OOZIE/4.2.0.3.0/configuration/oozie-env.xml | 10 +
.../RANGER/0.4.0/configuration/ranger-env.xml | 10 +
.../0.4.0/package/scripts/ranger_admin.py | 13 +-
.../1.0.0.3.0/configuration/ranger-env.xml | 10 +
.../0.5.0.2.3/configuration/kms-env.xml | 10 +
.../1.0.0.3.0/configuration/kms-env.xml | 10 +
.../RANGER_KMS/1.0.0.3.0/service_advisor.py | 3 +-
.../SPARK/1.2.1/configuration/spark-env.xml | 10 +
.../SPARK/2.2.0/configuration/livy-env.xml | 10 +
.../SPARK/2.2.0/configuration/spark-env.xml | 10 +
.../SPARK2/2.0.0/configuration/spark2-env.xml | 10 +
.../SQOOP/1.4.4.2.0/configuration/sqoop-env.xml | 6 +
.../SQOOP/1.4.4.3.0/configuration/sqoop-env.xml | 6 +
.../STORM/0.9.1/configuration/storm-env.xml | 6 +
.../STORM/1.0.1.3.0/configuration/storm-env.xml | 6 +
.../0.15.0/configuration/superset-env.xml | 6 +
.../TEZ/0.4.0.2.1/configuration/tez-env.xml | 10 +
.../TEZ/0.9.0.3.0/configuration/tez-env.xml | 10 +
.../configuration-mapred/mapred-env.xml | 6 +
.../YARN/2.1.0.2.0/configuration/yarn-env.xml | 6 +
.../configuration-mapred/mapred-env.xml | 6 +
.../YARN/3.0.0.3.0/configuration/yarn-env.xml | 6 +
.../0.6.0/configuration/zeppelin-env.xml | 10 +
.../0.7.0/configuration/zeppelin-env.xml | 10 +
.../ZEPPELIN/0.7.0/package/scripts/master.py | 25 +-
.../3.4.5/configuration/zookeeper-env.xml | 6 +
.../src/main/resources/configuration-schema.xsd | 2 +-
.../custom_actions/scripts/install_packages.py | 35 +-
ambari-server/src/main/resources/kerberos.json | 79 ++
.../main/resources/scripts/Ambaripreupload.py | 4 +-
.../scripts/post-user-creation-hook.sh | 2 +-
.../stack-hooks/after-INSTALL/scripts/hook.py | 37 +
.../stack-hooks/after-INSTALL/scripts/params.py | 108 +++
.../scripts/shared_initialization.py | 132 ++++
.../before-ANY/files/changeToSecureUid.sh | 64 ++
.../stack-hooks/before-ANY/scripts/hook.py | 36 +
.../stack-hooks/before-ANY/scripts/params.py | 254 ++++++
.../before-ANY/scripts/shared_initialization.py | 273 +++++++
.../stack-hooks/before-INSTALL/scripts/hook.py | 37 +
.../before-INSTALL/scripts/params.py | 115 +++
.../scripts/repo_initialization.py | 75 ++
.../scripts/shared_initialization.py | 37 +
.../stack-hooks/before-RESTART/scripts/hook.py | 29 +
.../before-START/files/checkForFormat.sh | 65 ++
.../before-START/files/fast-hdfs-resource.jar | Bin 0 -> 28296600 bytes
.../before-START/files/task-log4j.properties | 134 ++++
.../before-START/files/topology_script.py | 66 ++
.../before-START/scripts/custom_extensions.py | 173 ++++
.../stack-hooks/before-START/scripts/hook.py | 43 +
.../stack-hooks/before-START/scripts/params.py | 380 +++++++++
.../before-START/scripts/rack_awareness.py | 48 ++
.../scripts/shared_initialization.py | 256 ++++++
.../templates/commons-logging.properties.j2 | 43 +
.../templates/exclude_hosts_list.j2 | 21 +
.../templates/hadoop-metrics2.properties.j2 | 107 +++
.../before-START/templates/health_check.j2 | 81 ++
.../templates/include_hosts_list.j2 | 21 +
.../templates/topology_mappings.data.j2 | 24 +
.../HDP/2.0.6/configuration/cluster-env.xml | 10 +
.../2.0.6/hooks/after-INSTALL/scripts/hook.py | 37 -
.../2.0.6/hooks/after-INSTALL/scripts/params.py | 115 ---
.../scripts/shared_initialization.py | 132 ----
.../hooks/before-ANY/files/changeToSecureUid.sh | 64 --
.../HDP/2.0.6/hooks/before-ANY/scripts/hook.py | 36 -
.../2.0.6/hooks/before-ANY/scripts/params.py | 277 -------
.../before-ANY/scripts/shared_initialization.py | 281 -------
.../2.0.6/hooks/before-INSTALL/scripts/hook.py | 37 -
.../hooks/before-INSTALL/scripts/params.py | 115 ---
.../scripts/repo_initialization.py | 75 --
.../scripts/shared_initialization.py | 37 -
.../2.0.6/hooks/before-RESTART/scripts/hook.py | 29 -
.../hooks/before-START/files/checkForFormat.sh | 65 --
.../before-START/files/fast-hdfs-resource.jar | Bin 28296600 -> 0 bytes
.../before-START/files/task-log4j.properties | 134 ----
.../hooks/before-START/files/topology_script.py | 66 --
.../before-START/scripts/custom_extensions.py | 173 ----
.../2.0.6/hooks/before-START/scripts/hook.py | 43 -
.../2.0.6/hooks/before-START/scripts/params.py | 387 ---------
.../before-START/scripts/rack_awareness.py | 48 --
.../scripts/shared_initialization.py | 256 ------
.../templates/commons-logging.properties.j2 | 43 -
.../templates/exclude_hosts_list.j2 | 21 -
.../templates/hadoop-metrics2.properties.j2 | 107 ---
.../before-START/templates/health_check.j2 | 81 --
.../templates/include_hosts_list.j2 | 21 -
.../templates/topology_mappings.data.j2 | 24 -
.../resources/stacks/HDP/2.0.6/kerberos.json | 79 --
.../stacks/HDP/2.0.6/services/stack_advisor.py | 562 +------------
.../resources/stacks/HDP/2.0.6/widgets.json | 95 ---
.../services/HBASE/configuration/hbase-env.xml | 6 +
.../stacks/HDP/2.2/services/stack_advisor.py | 1 -
.../services/ECS/package/scripts/ecs_client.py | 2 +-
.../services/ECS/package/scripts/params.py | 2 +-
.../services/ATLAS/configuration/atlas-env.xml | 6 +
.../HIVE/configuration/llap-daemon-log4j.xml | 2 +-
.../services/SPARK/configuration/livy-env.xml | 10 +
.../stacks/HDP/2.5/services/stack_advisor.py | 3 +-
.../stacks/HDP/2.6/kerberos_preconfigure.json | 9 +
.../stacks/HDP/2.6/services/DRUID/metainfo.xml | 5 +-
.../HIVE/configuration/tez-interactive-site.xml | 6 +
.../services/SPARK/configuration/livy-env.xml | 10 +
.../services/YARN/configuration/yarn-site.xml | 18 +
.../ZEPPELIN/configuration/zeppelin-env.xml | 10 +
.../stacks/HDP/2.6/services/stack_advisor.py | 28 +-
.../HDP/3.0/configuration/cluster-env.xml | 10 +
.../HDP/3.0/hooks/after-INSTALL/scripts/hook.py | 37 -
.../3.0/hooks/after-INSTALL/scripts/params.py | 109 ---
.../scripts/shared_initialization.py | 140 ----
.../hooks/before-ANY/files/changeToSecureUid.sh | 53 --
.../HDP/3.0/hooks/before-ANY/scripts/hook.py | 36 -
.../HDP/3.0/hooks/before-ANY/scripts/params.py | 259 ------
.../before-ANY/scripts/shared_initialization.py | 239 ------
.../3.0/hooks/before-INSTALL/scripts/hook.py | 37 -
.../3.0/hooks/before-INSTALL/scripts/params.py | 115 ---
.../scripts/repo_initialization.py | 76 --
.../scripts/shared_initialization.py | 37 -
.../3.0/hooks/before-RESTART/scripts/hook.py | 29 -
.../hooks/before-START/files/checkForFormat.sh | 65 --
.../before-START/files/fast-hdfs-resource.jar | Bin 28296600 -> 0 bytes
.../before-START/files/task-log4j.properties | 134 ----
.../hooks/before-START/files/topology_script.py | 66 --
.../HDP/3.0/hooks/before-START/scripts/hook.py | 40 -
.../3.0/hooks/before-START/scripts/params.py | 364 ---------
.../before-START/scripts/rack_awareness.py | 47 --
.../scripts/shared_initialization.py | 249 ------
.../templates/commons-logging.properties.j2 | 43 -
.../templates/exclude_hosts_list.j2 | 21 -
.../templates/hadoop-metrics2.properties.j2 | 107 ---
.../before-START/templates/health_check.j2 | 81 --
.../templates/include_hosts_list.j2 | 21 -
.../templates/topology_mappings.data.j2 | 24 -
.../main/resources/stacks/HDP/3.0/kerberos.json | 79 --
.../main/resources/stacks/HDP/3.0/widgets.json | 95 ---
.../src/main/resources/upgrade-pack.xsd | 9 +
ambari-server/src/main/resources/widgets.json | 95 +++
.../server/api/services/AmbariMetaInfoTest.java | 49 +-
.../AmbariManagementControllerImplTest.java | 6 +-
.../AmbariManagementControllerTest.java | 17 +
.../internal/UpgradeResourceProviderTest.java | 33 +-
.../upgrade/StackVersionListenerTest.java | 1 -
.../apache/ambari/server/orm/OrmTestHelper.java | 2 +-
.../FixCapacitySchedulerOrderingPolicyTest.java | 2 +-
.../upgrades/FixOozieAdminUsersTest.java | 2 +-
.../upgrades/FixYarnWebServiceUrlTest.java | 2 +-
.../HBaseEnvMaxDirectMemorySizeActionTest.java | 2 +-
.../upgrades/HiveEnvClasspathActionTest.java | 2 +-
.../upgrades/HiveZKQuorumConfigActionTest.java | 2 +-
.../upgrades/KerberosKeytabsActionTest.java | 17 +
.../upgrades/RangerConfigCalculationTest.java | 2 +-
.../RangerKerberosConfigCalculationTest.java | 2 +-
.../upgrades/RangerKmsProxyConfigTest.java | 2 +-
.../RangerUsersyncConfigCalculationTest.java | 2 +-
.../RangerWebAlertConfigActionTest.java | 6 +-
.../SparkShufflePropertyConfigTest.java | 2 +-
.../UpgradeUserKerberosDescriptorTest.java | 4 +-
.../ambari/server/stack/StackManagerTest.java | 8 -
.../state/kerberos/KerberosDescriptorTest.java | 56 ++
.../KerberosDescriptorUpdateHelperTest.java | 1 +
.../kerberos/VariableReplacementHelperTest.java | 35 +-
.../svccomphost/ServiceComponentHostTest.java | 102 ++-
.../topology/AsyncCallableServiceTest.java | 89 +--
.../ClusterDeployWithStartOnlyTest.java | 6 +
...InstallWithoutStartOnComponentLevelTest.java | 6 +
.../ClusterInstallWithoutStartTest.java | 6 +
.../topology/ConfigureClusterTaskTest.java | 64 +-
.../server/topology/TopologyManagerTest.java | 14 +-
.../server/upgrade/UpgradeCatalog260Test.java | 14 +
.../src/test/python/TestAmbariServer.py | 4 +-
ambari-server/src/test/python/TestMpacks.py | 12 +-
.../src/test/python/TestResourceFilesKeeper.py | 1 +
.../src/test/python/TestUpgradeSummary.py | 6 +-
.../AMBARI_METRICS/test_service_advisor.py | 596 ++++++++++++++
.../HIVE/test_jdbc_driver_config.py | 18 +-
.../RANGER/test_db_flavor_config.py | 17 +-
.../RANGER_KMS/test_db_flavor_config.py | 17 +-
.../SQOOP/test_jdbc_driver_config.py | 16 +-
.../configs/ranger_admin_default.json | 55 ++
.../custom_actions/TestInstallPackages.py | 50 +-
.../stacks/2.0.6/HBASE/test_hbase_client.py | 1 -
.../stacks/2.0.6/HBASE/test_hbase_master.py | 6 +-
.../2.0.6/HBASE/test_phoenix_queryserver.py | 7 +
.../python/stacks/2.0.6/HDFS/test_datanode.py | 38 +-
.../python/stacks/2.0.6/HDFS/test_namenode.py | 5 +-
.../stacks/2.0.6/HIVE/test_hive_metastore.py | 42 +-
.../stacks/2.0.6/HIVE/test_hive_server.py | 158 ++--
.../2.0.6/HIVE/test_hive_service_check.py | 4 +-
.../stacks/2.0.6/HIVE/test_webhcat_server.py | 41 +-
.../stacks/2.0.6/OOZIE/test_oozie_server.py | 9 +
.../2.0.6/OOZIE/test_oozie_service_check.py | 5 +-
.../stacks/2.0.6/YARN/test_historyserver.py | 5 +-
.../stacks/2.0.6/YARN/test_mapreduce2_client.py | 1 +
.../stacks/2.0.6/common/test_stack_advisor.py | 576 --------------
.../python/stacks/2.0.6/configs/default.json | 1 +
.../hooks/after-INSTALL/test_after_install.py | 88 +--
.../2.0.6/hooks/before-ANY/test_before_any.py | 24 +-
.../hooks/before-INSTALL/test_before_install.py | 14 +-
.../hooks/before-START/test_before_start.py | 21 +-
.../stacks/2.1/FALCON/test_falcon_server.py | 26 +-
.../stacks/2.1/HIVE/test_hive_metastore.py | 54 +-
.../stacks/2.2/PIG/test_pig_service_check.py | 13 +
.../stacks/2.2/SPARK/test_job_history_server.py | 18 +-
.../stacks/2.2/common/test_conf_select.py | 13 +-
.../stacks/2.2/common/test_stack_advisor.py | 511 ------------
.../2.2/common/test_stack_advisor_perf.py | 66 +-
.../stacks/2.3/MAHOUT/test_mahout_client.py | 2 +-
.../2.3/MAHOUT/test_mahout_service_check.py | 28 +-
.../2.3/SPARK/test_spark_thrift_server.py | 8 +-
.../stacks/2.3/common/test_stack_advisor.py | 2 +-
.../stacks/2.5/RANGER_KMS/test_kms_server.py | 24 +-
.../python/stacks/2.5/SPARK/test_spark_livy.py | 16 +-
.../stacks/2.5/ZEPPELIN/test_zeppelin_060.py | 20 +-
.../stacks/2.5/configs/ranger-kms-secured.json | 6 +-
.../test/python/stacks/2.6/DRUID/test_druid.py | 22 +-
.../stacks/2.6/SPARK2/test_spark_livy2.py | 16 +-
.../stacks/2.6/ZEPPELIN/test_zeppelin_070.py | 236 +++---
.../stacks/2.6/common/test_stack_advisor.py | 190 ++++-
.../src/test/python/stacks/utils/RMFTestCase.py | 43 +-
ambari-server/src/test/resources/kerberos.json | 42 +
.../resources/stacks/HDP/2.0.8/kerberos.json | 42 -
ambari-server/src/test/resources/widgets.json | 95 +++
ambari-web/app/controllers/installer.js | 8 +-
.../journalNode/progress_controller.js | 4 +-
.../journalNode/step4_controller.js | 6 +-
.../nameNode/step5_controller.js | 6 +-
.../highAvailability/progress_controller.js | 4 +-
.../progress_popup_controller.js | 2 +-
.../main/admin/kerberos/step2_controller.js | 7 +-
.../main/admin/stack_and_upgrade_controller.js | 4 +-
ambari-web/app/controllers/main/host/details.js | 2 -
.../controllers/main/service/info/summary.js | 1 +
.../main/service/reassign/step4_controller.js | 2 -
.../app/controllers/wizard/step6_controller.js | 15 +-
.../app/controllers/wizard/step8_controller.js | 15 +-
.../app/mappers/repository_version_mapper.js | 2 +-
ambari-web/app/messages.js | 8 +-
.../app/mixins/common/configs/configs_saver.js | 26 +-
.../main/service/configs/config_overridable.js | 1 -
.../models/stack_version/repository_version.js | 2 +-
ambari-web/app/routes/add_kerberos_routes.js | 22 +-
ambari-web/app/styles/alerts.less | 14 +-
ambari-web/app/styles/application.less | 7 +-
ambari-web/app/styles/bootstrap_overrides.less | 4 +
ambari-web/app/styles/dashboard.less | 22 +-
ambari-web/app/styles/modal_popups.less | 2 +-
ambari-web/app/styles/stack_versions.less | 13 +-
.../app/styles/theme/bootstrap-ambari.css | 17 +-
ambari-web/app/styles/top-nav.less | 5 +-
ambari-web/app/styles/wizard.less | 9 +
ambari-web/app/templates/common/breadcrumbs.hbs | 4 +-
.../stack_upgrade/stack_upgrade_wizard.hbs | 2 +-
.../main/service/info/service_alert_popup.hbs | 6 +-
.../app/templates/main/service/info/summary.hbs | 21 +-
.../service/info/summary/master_components.hbs | 4 -
.../templates/main/service/services/hdfs.hbs | 38 +-
ambari-web/app/templates/wizard/step6.hbs | 2 +-
ambari-web/app/templates/wizard/step9.hbs | 12 +-
ambari-web/app/utils/ajax/ajax.js | 2 -
.../stack_upgrade/upgrade_version_box_view.js | 70 +-
.../upgrade_version_column_view.js | 2 +-
.../admin/stack_upgrade/upgrade_wizard_view.js | 26 +-
.../app/views/main/service/info/summary.js | 8 +
.../app/views/main/service/services/hdfs.js | 2 -
ambari-web/test/controllers/installer_test.js | 37 +-
.../journalNode/progress_controller_test.js | 2 -
.../progress_controller_test.js | 3 -
.../progress_popup_controller_test.js | 4 +-
.../admin/kerberos/step2_controller_test.js | 9 +-
.../mixins/common/configs/configs_saver_test.js | 28 +-
.../stack_version/repository_version_test.js | 12 +-
.../upgrade_version_box_view_test.js | 149 ++--
.../upgrade_version_column_view_test.js | 6 +
.../stack_upgrade/upgrade_wizard_view_test.js | 38 +
.../src/main/assemblies/hdf-ambari-mpack.xml | 1 +
.../HIVE/package/scripts/hive_client.py | 2 -
.../HIVE/package/scripts/hive_metastore.py | 1 -
.../HIVE/package/scripts/hive_server.py | 2 +-
.../package/scripts/hive_server_interactive.py | 1 -
.../HIVE/package/scripts/webhcat_server.py | 2 -
.../scripts/application_timeline_server.py | 1 -
.../YARN/package/scripts/historyserver.py | 1 -
.../YARN/package/scripts/mapreduce2_client.py | 2 -
.../YARN/package/scripts/nodemanager.py | 1 -
.../YARN/package/scripts/resourcemanager.py | 1 -
.../YARN/package/scripts/yarn_client.py | 1 -
.../ui/hive-web/app/adapters/application.js | 18 +
.../app/controllers/visualization-ui.js | 10 +-
.../ui/hive-web/app/utils/constants.js | 1 +
.../savedQueries/SavedQueryResourceManager.java | 17 +-
.../resources/ui/app/components/job-item.js | 49 +-
.../resources/ui/app/components/jobs-browser.js | 3 +
.../ui/app/components/query-result-table.js | 2 +-
.../src/main/resources/ui/app/routes/jobs.js | 11 +
.../main/resources/ui/app/routes/queries/new.js | 2 +
.../resources/ui/app/routes/queries/query.js | 240 ++++--
.../src/main/resources/ui/app/services/jobs.js | 31 +-
.../src/main/resources/ui/app/styles/app.scss | 8 +
.../ui/app/templates/components/job-item.hbs | 2 +-
.../app/templates/components/jobs-browser.hbs | 2 +-
.../main/resources/ui/app/templates/jobs.hbs | 1 +
.../hive20/src/main/resources/ui/yarn.lock | 2 +-
.../apache/ambari/view/utils/hdfs/HdfsApi.java | 2 +-
614 files changed, 13386 insertions(+), 14066 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/assemblies/server.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java
index a5b3622,9f8a095..5cc5dd5
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java
@@@ -39,9 -39,9 +39,12 @@@ public class StatusCommand extends Agen
@SerializedName("serviceName")
private String serviceName;
+ @SerializedName("serviceType")
+ private String serviceType;
+
+ @SerializedName("role")
+ private String role;
+
@SerializedName("componentName")
private String componentName;
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
index 8883ced,46ee65a..c07df43
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java
@@@ -134,8 -136,9 +145,10 @@@ public class AmbariMetaInfo
private File commonServicesRoot;
private File extensionsRoot;
private File serverVersionFile;
+ private File commonWidgetsDescriptorFile;
private File customActionRoot;
+ private File mpacksV2Staging;
+ private String commonKerberosDescriptorFileLocation;
private Map<String, VersionDefinitionXml> versionDefinitions = null;
@@@ -243,9 -228,8 +256,11 @@@
customActionRoot = new File(conf.getCustomActionDefinitionPath());
+ String mpacksV2StagingPath = conf.getMpacksV2StagingPath();
+ mpacksV2Staging = new File(mpacksV2StagingPath);
+
+ commonKerberosDescriptorFileLocation = new File(conf.getResourceDirPath(), KERBEROS_DESCRIPTOR_FILE_NAME).getAbsolutePath();
+ commonWidgetsDescriptorFile = new File(conf.getResourceDirPath(), WIDGETS_DESCRIPTOR_FILE_NAME);
}
/**
@@@ -1414,10 -1360,9 +1425,10 @@@
/**
* Ensures that the map of version definition files is populated
*/
- private void ensureVersionDefinitions() {
+ private synchronized void ensureVersionDefinitions() {
if (null != versionDefinitions) {
- return;
+ if(versionDefinitions.size() > 0)
+ return;
}
versionDefinitions = new HashMap<>();
@@@ -1519,6 -1446,9 +1530,10 @@@
}
return null;
+
}
+
+ public File getCommonWidgetsDescriptorFile() {
+ return commonWidgetsDescriptorFile;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
index c56483d,dc97871..a4339ab
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java
@@@ -71,9 -72,9 +72,10 @@@ import org.apache.ambari.server.control
import org.apache.ambari.server.controller.internal.KerberosDescriptorResourceProvider;
import org.apache.ambari.server.controller.internal.MemberResourceProvider;
import org.apache.ambari.server.controller.internal.RepositoryVersionResourceProvider;
+import org.apache.ambari.server.controller.internal.ServiceGroupResourceProvider;
import org.apache.ambari.server.controller.internal.ServiceResourceProvider;
import org.apache.ambari.server.controller.internal.UpgradeResourceProvider;
+ import org.apache.ambari.server.controller.internal.ViewInstanceResourceProvider;
import org.apache.ambari.server.controller.logging.LoggingRequestHelperFactory;
import org.apache.ambari.server.controller.logging.LoggingRequestHelperFactoryImpl;
import org.apache.ambari.server.controller.metrics.MetricPropertyProviderFactory;
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/controller/ResourceProviderFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultProviderModule.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/HostVersionOutOfSyncListener.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackGenerator.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackGenerator.java
index 5d4b832,0000000..130ff62
mode 100644,000000..100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackGenerator.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackGenerator.java
@@@ -1,561 -1,0 +1,544 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.mpack;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.Marshaller;
+
+import org.apache.ambari.server.stack.ServiceDirectory;
+import org.apache.ambari.server.stack.StackDirectory;
+import org.apache.ambari.server.stack.StackManager;
+import org.apache.ambari.server.state.Mpack;
+import org.apache.ambari.server.state.Packlet;
+import org.apache.ambari.server.state.PropertyInfo;
+import org.apache.ambari.server.state.QuickLinksConfigurationInfo;
+import org.apache.ambari.server.state.ServiceInfo;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.StackInfo;
+import org.apache.ambari.server.state.ThemeInfo;
+import org.apache.ambari.server.state.quicklinks.QuickLinks;
+import org.apache.ambari.server.state.repository.ManifestServiceInfo;
+import org.apache.ambari.server.state.repository.VersionDefinitionXml;
+import org.apache.ambari.server.state.stack.ConfigurationXml;
+import org.apache.ambari.server.state.stack.RepositoryXml;
+import org.apache.ambari.server.state.stack.ServiceMetainfoXml;
+import org.apache.ambari.server.state.stack.StackMetainfoXml;
+import org.apache.ambari.server.state.stack.StackRoleCommandOrder;
+import org.apache.ambari.server.state.theme.Theme;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+/**
+ * Generate management pack from stack definition
+ */
+public class MpackGenerator {
+ private static final Logger LOG = LoggerFactory.getLogger
+ (MpackGenerator.class);
+
+ private static final ObjectMapper mapper = new ObjectMapper();
+ private File commonServicesRoot;
+ private File stackRoot;
+ private File mpacksRoot;
+ private StackId srcStackId;
+ private StackId dstStackId;
+ private StackManager stackManager;
+ private VersionDefinitionXml vdf;
+ private Gson gson = new GsonBuilder().setPrettyPrinting().create();
+
+ /**
+ * {@link MpackGenerator} constructor
+ * @param stackRoot Stack root directory
+ * @param commonServicesRoot Common services root directory
+ * @param srcStackId Source stack id
+ * @param vdfUrl Version Definition File
+ * @param mpacksRoot Management pack root directory
+ * @throws Exception
+ */
+ public MpackGenerator(File stackRoot, File commonServicesRoot, StackId srcStackId, URL vdfUrl, File mpacksRoot)
+ throws Exception {
+ this.commonServicesRoot = commonServicesRoot;
+ this.stackRoot = stackRoot;
+ this.mpacksRoot = mpacksRoot;
+ this.srcStackId = srcStackId;
+ this.stackManager = new StackManager(stackRoot, commonServicesRoot, false);
+ this.vdf = VersionDefinitionXml.load(vdfUrl);
+ this.dstStackId = new StackId(vdf.release.stackId);
+ }
+
+ /**
+ * Generate management pack
+ * @throws Exception
+ */
+ public void generateMpack() throws Exception {
+ String dstStackName = dstStackId.getStackName();
+ String version = vdf.release.version;
+ String build = vdf.release.build;
+
+ System.out.println("===========================================================");
+ System.out.println("Source Stack Id: " + srcStackId);
+ System.out.println("Destination Stack Id: " + dstStackId);
+ System.out.println("===========================================================");
+
+ String mpackName = dstStackName.toLowerCase() + "-ambari-mpack" + "-" + version + "-" + build;
+ File mpackRootDir = new File(mpacksRoot.getAbsolutePath() + File.separator + mpackName);
+ if (!mpacksRoot.exists()) {
+ mpacksRoot.mkdirs();
+ }
+ if (mpackRootDir.exists()) {
+ FileUtils.deleteDirectory(mpackRootDir);
+ }
+ mpackRootDir.mkdir();
+
+ File mpackPackletsDir = new File(mpackRootDir.getAbsolutePath() + File.separator + "packlets");
+ if (mpackPackletsDir.exists()) {
+ mpackPackletsDir.delete();
+ }
+ mpackPackletsDir.mkdir();
+
+ StackInfo srcStackInfo = stackManager.getStack(srcStackId.getStackName(), srcStackId.getStackVersion());
+ StackRoleCommandOrder stackRoleCommandOrder = srcStackInfo.getRoleCommandOrder();
+ FileWriter stackRCOFile = new FileWriter(
+ mpackRootDir.getAbsolutePath() + File.separator + StackDirectory.RCO_FILE_NAME);
+ mapper.writerWithDefaultPrettyPrinter().writeValue(stackRCOFile, stackRoleCommandOrder.getContent());
+
+ // Export stack configs
+ File stackConfigDir = new File(
+ mpackRootDir.getAbsolutePath() + File.separator + StackDirectory.SERVICE_CONFIG_FOLDER_NAME);
+ exportConfigs(srcStackInfo.getProperties(), stackConfigDir);
+
- // Export widgets.json
- exportFile(new File(srcStackInfo.getWidgetsDescriptorFileLocation()), mpackRootDir);
-
- // Export kerberos.json
- exportFile(new File(srcStackInfo.getKerberosDescriptorFileLocation()), mpackRootDir);
-
+ // Export repoinfo.xml
+ RepositoryXml repositoryXml = srcStackInfo.getRepositoryXml();
+ JAXBContext ctx = JAXBContext.newInstance(RepositoryXml.class);
+ Marshaller marshaller = ctx.createMarshaller();
+ marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
+ File reposDir = new File(mpackRootDir.getAbsolutePath() + File.separator + "repos");
+ if (!reposDir.exists()) {
+ reposDir.mkdir();
+ }
+ FileOutputStream repoXmlFileStream = new FileOutputStream(
+ reposDir.getAbsolutePath() + File.separator + "repoinfo.xml");
+ marshaller.marshal(repositoryXml, repoXmlFileStream);
+ repoXmlFileStream.flush();
+ repoXmlFileStream.close();
+
+ // Copy JSON cluster property files (example: stacks/HDP/2.0.6/properties/stack_tools.json)
+ File destPropertiesDir = new File(mpackRootDir.getAbsoluteFile() + File.separator + "properties");
+ if(!destPropertiesDir.exists()) {
+ destPropertiesDir.mkdir();
+ }
+ String srcStackName = srcStackId.getStackName();
+ String currentStackVersion = srcStackId.getStackVersion();
+
+ while (!StringUtils.isEmpty(currentStackVersion)) {
+ StackInfo currentStackInfo = stackManager.getStack(srcStackName, currentStackVersion);
+ File srcPropertiesDir = new File(stackRoot.getAbsolutePath() + File.separator + srcStackName + File.separator + currentStackVersion + File.separator + "properties");
+ if (srcPropertiesDir.exists() && srcPropertiesDir.isDirectory()) {
+ for (File srcPropertiesFile : srcPropertiesDir.listFiles()) {
+ File destPropertiesFile = new File(destPropertiesDir.getAbsolutePath() + File.separator + srcPropertiesFile.getName());
+ if (!destPropertiesFile.exists()) {
+ FileUtils.copyFile(srcPropertiesFile, destPropertiesFile);
+ }
+ }
+ }
+ currentStackVersion = currentStackInfo.getParentStackVersion();
+ }
+
- // Copy stack hooks folder
- String srcStackHooksFolder = srcStackInfo.getStackHooksFolder();
- srcStackHooksFolder = stackRoot.getAbsolutePath() + File.separator + srcStackHooksFolder;
- File srcStackHooksFile = new File(srcStackHooksFolder);
- if (srcStackHooksFile != null && srcStackHooksFile.exists()) {
- File destStackHooksFile = new File(
- mpackRootDir.getAbsolutePath() + File.separator
- + srcStackHooksFile.getName());
- FileUtils.copyDirectory(srcStackHooksFile, destStackHooksFile);
- }
-
+ // Export stack metainfo.xml
+ String parentStackVersion = srcStackInfo.getParentStackVersion();
+ StackMetainfoXml stackMetainfoXml = new StackMetainfoXml();
+ stackMetainfoXml.setMinJdk(srcStackInfo.getMinJdk());
+ stackMetainfoXml.setMaxJdk(srcStackInfo.getMaxJdk());
+ StackMetainfoXml.Version ver = new StackMetainfoXml.Version();
+ ver.setActive(srcStackInfo.isActive());
+ stackMetainfoXml.setVersion(ver);
+ ctx = JAXBContext.newInstance(StackMetainfoXml.class);
+ marshaller = ctx.createMarshaller();
+ marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
+ FileOutputStream stackMetainfoFileStream = new FileOutputStream(
+ mpackRootDir.getAbsolutePath() + File.separator + "metainfo.xml");
+ marshaller.marshal(stackMetainfoXml, stackMetainfoFileStream);
+ stackMetainfoFileStream.flush();
+ stackMetainfoFileStream.close();
+
+ // Export stack advisors
+ File stackAdvisorsDir = new File(mpackRootDir.getAbsolutePath() + File.separator + "stack-advisor");
+ if(!stackAdvisorsDir.exists()) {
+ stackAdvisorsDir.mkdir();
+ }
+
+ currentStackVersion = srcStackId.getStackVersion();
+ String baseStackAdvisor = null;
+ String baseStackAdvisorModule = null;
+ while (!StringUtils.isEmpty(currentStackVersion)) {
+ // Copy all inherited stack advisors from source stack to "stack-advisor" folder
+ StackInfo currentStackInfo = stackManager.getStack(srcStackName, currentStackVersion);
+ File srcStackAdvisor = new File(stackRoot.getAbsolutePath() + File.separator + srcStackName + File.separator + currentStackVersion + File.separator + "services" + File.separator + "stack_advisor.py");
+ if(srcStackAdvisor.exists()) {
+ if(baseStackAdvisor == null) {
+ baseStackAdvisor = srcStackName.toUpperCase() + currentStackVersion.replace(".", "") + "StackAdvisor";
+ baseStackAdvisorModule = "stack_advisor_" + srcStackName.toLowerCase() + currentStackVersion.replace(".", "");
+ }
+ File dstStackAdvisor = new File(
+ stackAdvisorsDir.getAbsolutePath() + File.separator + "stack_advisor_" + srcStackName.toLowerCase()
+ + currentStackVersion.replace(".", "") + ".py");
+ FileUtils.copyFile(srcStackAdvisor, dstStackAdvisor);
+ }
+ currentStackVersion = currentStackInfo.getParentStackVersion();
+ }
+ if(baseStackAdvisor != null) {
+ File mpackServicesDir = new File(mpackRootDir.getAbsolutePath() + File.separator + "services");
+ if (!mpackServicesDir.exists()) {
+ mpackServicesDir.mkdir();
+ }
+ String mpackStackAdvisorName = dstStackId.getStackName().toUpperCase() + dstStackId.getStackVersion().replace(".", "") + "StackAdvisor";
+ if(baseStackAdvisor.equalsIgnoreCase(mpackStackAdvisorName)) {
+ // Use top level stack advisor from source stack as mpack stack advisor
+ String srcPath = stackAdvisorsDir.getAbsolutePath() + File.separator + baseStackAdvisorModule + ".py";
+ String dstPath = mpackServicesDir.getAbsolutePath() + File.separator + "stack_advisor.py";
+ Files.move(Paths.get(srcPath), Paths.get(dstPath));
+ } else {
+ // Create mpack stack advisor that inherits from top level stack advisor from source stack
+ FileWriter fileWriter = new FileWriter(
+ mpackServicesDir.getAbsolutePath() + File.separator + "stack_advisor.py");
+ BufferedWriter bw = new BufferedWriter(fileWriter);
+ bw.write("from " + baseStackAdvisorModule + " import *");
+ bw.newLine();
+ bw.write("class " + mpackStackAdvisorName + "(" + baseStackAdvisor + ")");
+ bw.newLine();
+ bw.write(" pass");
+ bw.newLine();
+ bw.flush();
+ fileWriter.flush();
+ bw.close();
+ fileWriter.close();
+ }
+ }
+
+ Mpack mpack = new Mpack();
+ mpack.setName(dstStackName);
+ mpack.setVersion(vdf.release.version);
+ mpack.setBuildNumber(vdf.release.getFullVersion());
+ mpack.setStackId(dstStackId.getStackId());
+ mpack.setDescription(dstStackName + " Ambari Management Pack");
+ Map<String, String> prereqs = new HashMap<>();
+ prereqs.put("min-ambari-version", "3.0.0.0");
+ mpack.setPrerequisites(prereqs);
+ List<Packlet> packlets = new ArrayList<>();
+ mpack.setPacklets(packlets);
+
+ for (ManifestServiceInfo manifestServiceInfo : vdf.getStackServices(srcStackInfo, true /* skipMissingServices = true */)) {
+ ServiceInfo serviceInfo = srcStackInfo.getService(manifestServiceInfo.getName());
+ String serviceName = manifestServiceInfo.getName();
+ String serviceVersion = (String) manifestServiceInfo.getVersions().toArray()[0];
+ ServiceInfo clonedServiceInfo = (ServiceInfo) serviceInfo.clone();
+ clonedServiceInfo.setVersion(serviceVersion);
+ clonedServiceInfo.setParent(null);
+ if (serviceInfo.getMetricsFile() != null) {
+ clonedServiceInfo.setMetricsFileName(serviceInfo.getMetricsFile().getName());
+ }
+ if( serviceInfo.getWidgetsDescriptorFile() != null) {
+ clonedServiceInfo.setWidgetsFileName(serviceInfo.getWidgetsDescriptorFile().getName());
+ }
+
+ System.out.println("Processing service=" + serviceInfo.getName() + ", version=" + serviceVersion);
+ System.out.println("Service Parent : " + serviceInfo.getParent());
+ String packletDirName = serviceName + "-packlet-" + serviceVersion;
+ String packletTarName = packletDirName + ".tar.gz";
+ File packletDir = new File(
+ mpackPackletsDir.getAbsolutePath() + File.separator + packletDirName);
+ if (!packletDir.exists()) {
+ packletDir.mkdir();
+ }
+ Packlet packlet = new Packlet();
+ packlet.setType(Packlet.PackletType.SERVICE_PACKLET);
+ packlet.setName(serviceName);
+ packlet.setVersion(serviceVersion);
+ packlet.setServiceId(serviceVersion);
+ packlet.setSourceLocation("packlets" + File.separator + packletTarName);
+ packlets.add(packlet);
+
+ // Export service metainfo.xml
+ ServiceMetainfoXml serviceMetainfoXml = new ServiceMetainfoXml();
+ serviceMetainfoXml.setSchemaVersion(clonedServiceInfo.getSchemaVersion());
+ List<ServiceInfo> serviceInfos = Collections.singletonList(clonedServiceInfo);
+ serviceMetainfoXml.setServices(serviceInfos);
+ ctx = JAXBContext.newInstance(ServiceMetainfoXml.class);
+ marshaller = ctx.createMarshaller();
+ marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
+ FileOutputStream serviceMetainfoFileStream = new FileOutputStream(
+ packletDir.getAbsolutePath() + File.separator + "metainfo.xml");
+ marshaller.marshal(serviceMetainfoXml, serviceMetainfoFileStream);
+ serviceMetainfoFileStream.flush();
+ serviceMetainfoFileStream.close();
+
+ // Export mertrics.json
+ File srcMetricsFile = serviceInfo.getMetricsFile();
+ exportFile(srcMetricsFile, packletDir);
+
+ // Export widgets.json
+ File srcWidgetsFile = serviceInfo.getWidgetsDescriptorFile();
+ exportFile(srcWidgetsFile, packletDir);
+
+ // Export alerts.json
+ File srcAlertsFile = serviceInfo.getAlertsFile();
+ exportFile(srcAlertsFile, packletDir);
+
+ // Export kerberos.json
+ File srcKerberosFile = serviceInfo.getKerberosDescriptorFile();
+ exportFile(srcKerberosFile, packletDir);
+
+ // Export quicklinks
+ for (Map.Entry<String, QuickLinksConfigurationInfo> entry : serviceInfo.getQuickLinksConfigurationsMap()
+ .entrySet()) {
+ QuickLinksConfigurationInfo quickLinksConfigurationInfo = entry.getValue();
+ String quickLinksFileName = quickLinksConfigurationInfo.getFileName();
+ for (Map.Entry<String, QuickLinks> quickLinksEntry : quickLinksConfigurationInfo
+ .getQuickLinksConfigurationMap().entrySet()) {
+ File quickLinksDir = new File(
+ packletDir.getAbsolutePath() + File.separator + serviceInfo
+ .getQuickLinksConfigurationsDir());
+ if (!quickLinksDir.exists()) {
+ quickLinksDir.mkdir();
+ }
+ FileWriter quickLinksFileWriter = new FileWriter(
+ quickLinksDir.getAbsolutePath() + File.separator + quickLinksFileName, true);
+ mapper.writerWithDefaultPrettyPrinter()
+ .writeValue(quickLinksFileWriter, quickLinksEntry.getValue());
+ }
+ }
+
+ // Export themes
+ for (Map.Entry<String, ThemeInfo> entry : serviceInfo.getThemesMap().entrySet()) {
+ ThemeInfo themeInfo = entry.getValue();
+ String themeFileName = themeInfo.getFileName();
+ for (Map.Entry<String, Theme> themeEntry : themeInfo.getThemeMap().entrySet()) {
+ File themesDir = new File(
+ packletDir.getAbsolutePath() + File.separator + serviceInfo.getThemesDir());
+ if (!themesDir.exists()) {
+ themesDir.mkdir();
+ }
+ FileWriter themesFileWriter = new FileWriter(
+ themesDir.getAbsolutePath() + File.separator + themeFileName, true);
+ mapper.writerWithDefaultPrettyPrinter().writeValue(themesFileWriter, themeEntry.getValue());
+ }
+ }
+
+ // Export package folder
+ String srcPackageFolder = serviceInfo.getServicePackageFolder();
+ if (srcPackageFolder.startsWith("common-services")) {
+ srcPackageFolder = srcPackageFolder
+ .replace("common-services", commonServicesRoot.getAbsolutePath());
+ } else {
+ srcPackageFolder = srcPackageFolder.replace("stacks", stackRoot.getAbsolutePath());
+ }
+ File srcPackageFile = new File(srcPackageFolder);
+ if (srcPackageFile != null && srcPackageFile.exists()) {
+ File destPackageFile = new File(
+ packletDir.getAbsolutePath() + File.separator
+ + ServiceDirectory.PACKAGE_FOLDER_NAME);
+ FileUtils.copyDirectory(srcPackageFile, destPackageFile);
+ }
+
+ // Export merged configs
+ File configDir = new File(
+ packletDir.getAbsolutePath() + File.separator + serviceInfo.getConfigDir());
+ exportConfigs(serviceInfo.getProperties(), configDir);
+
+ // Copy service advisor
+ File srcServiceAdvisor = serviceInfo.getAdvisorFile();
+ File destServiceAdvisor = new File(packletDir.getAbsolutePath() + File.separator + "service_advisor.py");
+ if(srcServiceAdvisor != null && srcServiceAdvisor.exists()) {
+ FileUtils.copyFile(srcServiceAdvisor, destServiceAdvisor);
+ }
+
+ // TODO: Export upgrade packs
+
+ // Create packlet tarball
+ createTarGzip(packletDir.getAbsolutePath());
+ if(packletDir.exists()) {
+ FileUtils.deleteDirectory(packletDir);
+ }
+ }
+
+ // Create mpack.json
+ String mpackFilePath = mpackRootDir.getAbsolutePath() + File.separator + "mpack.json";
+ FileWriter mpackFileWriter = new FileWriter(mpackFilePath);
+ gson.toJson(mpack, Mpack.class, mpackFileWriter);
+ mpackFileWriter.flush();
+ mpackFileWriter.close();
+
+ // Create mpack tarball
+ createTarGzip(mpackRootDir.getAbsolutePath());
+ if(mpackRootDir.exists()) {
+ FileUtils.deleteDirectory(mpackRootDir);
+ }
+ }
+
+ public static void exportFile(File srcFile, File destRootDir) throws Exception {
+ if (srcFile != null && srcFile.exists()) {
+ Path srcPath = Paths.get(srcFile.getAbsolutePath());
+ Path destPath = Paths.get(
+ destRootDir.getAbsolutePath() + File.separator + srcFile.getName());
+ Files.copy(srcPath, destPath, StandardCopyOption.COPY_ATTRIBUTES,
+ StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+
+ public void createTarGzip(String path) throws FileNotFoundException, IOException {
+ File file = new File(path);
+ if(!file.exists()) {
+ throw new FileNotFoundException(path + " does not exist");
+ }
+ String parentDirName = file.getParent();
+ String tarGzPath = parentDirName + File.separator + file.getName() + ".tar.gz";
+ // Delete existing tarball
+ File f = new File(tarGzPath);
+ if(f.exists()) {
+ f.delete();
+ }
+ FileOutputStream fOut = null;
+ BufferedOutputStream bOut = null;
+ GzipCompressorOutputStream gzOut = null;
+ TarArchiveOutputStream tOut = null;
+ try {
+ fOut = new FileOutputStream(new File(tarGzPath));
+ bOut = new BufferedOutputStream(fOut);
+ gzOut = new GzipCompressorOutputStream(bOut);
+ tOut = new TarArchiveOutputStream(gzOut);
+ addFileToTarGz(tOut, path, "");
+ System.out.println("Compressed " + path + " -> " + tarGzPath);
+ } finally {
+ if(tOut != null) {
+ tOut.finish();
+ tOut.close();
+ }
+ if(gzOut != null) {
+ gzOut.close();
+ }
+ if(bOut != null) {
+ bOut.close();
+ }
+ if(fOut != null) {
+ fOut.close();
+ }
+ }
+
+ }
+
+ private void addFileToTarGz(TarArchiveOutputStream tOut, String path, String base)
+ throws IOException
+ {
+ File f = new File(path);
+ String entryName = base + f.getName();
+ TarArchiveEntry tarEntry = new TarArchiveEntry(f, entryName);
+ tOut.putArchiveEntry(tarEntry);
+
+ if (f.isFile()) {
+ IOUtils.copy(new FileInputStream(f), tOut);
+ tOut.closeArchiveEntry();
+ } else {
+ tOut.closeArchiveEntry();
+ File[] children = f.listFiles();
+ if (children != null) {
+ for (File child : children) {
+ addFileToTarGz(tOut, child.getAbsolutePath(), entryName + "/");
+ }
+ }
+ }
+ }
+
+ public static void exportConfigs(List<PropertyInfo> properties, File configDir) throws Exception {
+ if (!configDir.exists()) {
+ configDir.mkdir();
+ }
+
+ Map<String, List<PropertyInfo>> configFilesMap = new HashMap<>();
+ for (PropertyInfo propertyInfo : properties) {
+ String fileName = propertyInfo.getFilename();
+ if (!configFilesMap.containsKey(fileName)) {
+ configFilesMap.put(fileName, new ArrayList<PropertyInfo>());
+ }
+ configFilesMap.get(fileName).add(propertyInfo);
+ }
+
+ for (Map.Entry<String, List<PropertyInfo>> entry : configFilesMap.entrySet()) {
+ String fileName = entry.getKey();
+ ConfigurationXml configXml = new ConfigurationXml();
+ configXml.setProperties(entry.getValue());
+ JAXBContext ctx = JAXBContext.newInstance(ConfigurationXml.class);
+ Marshaller marshaller = ctx.createMarshaller();
+ marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
+ FileOutputStream configFileStream = new FileOutputStream(
+ configDir.getAbsolutePath() + File.separator + fileName);
+ marshaller.marshal(configXml, configFileStream);
+ configFileStream.flush();
+ configFileStream.close();
+ }
+ }
+
+ /**
+ * Main method for generating mpack
+ * @param args
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ System.out.println("Mpack Generator Started");
+ String stackDir = args[0];
+ String commonServicesDir = args[1];
+ String srcStack = args[2];
+ String vdfPath = args[3];
+ String mpacksDir = args[4];
+ MpackGenerator mpackGenerator = new MpackGenerator(
+ new File(stackDir),
+ new File(commonServicesDir),
+ new StackId(srcStack),
+ new URL(vdfPath),
+ new File(mpacksDir));
+ mpackGenerator.generateMpack();
+ System.out.println("Mpack Generator Finished");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
index da23ebc,eb6737a..9b740d0
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
@@@ -205,91 -193,6 +210,85 @@@ public class StackManager
populateDB(stackDao, extensionDao);
}
+ /***
+ * Constructor. Initialize StackManager for merging service definitions and creating management packs
+ * @param stackRoot
+ * @param commonServicesRoot
+ */
+ public StackManager(File stackRoot, File commonServicesRoot, boolean validate) throws AmbariException{
+ LOG.info("Initializing the stack manager...");
+
+ if (validate) {
+ validateStackDirectory(stackRoot);
+ validateCommonServicesDirectory(commonServicesRoot);
+ }
+
+ stackMap = new HashMap<>();
+
+ parseDirectories(stackRoot, commonServicesRoot, null);
+
+ fullyResolveCommonServices(stackModules, commonServiceModules, extensionModules);
+ fullyResolveExtensions(stackModules, commonServiceModules, extensionModules);
+ fullyResolveStacks(stackModules, commonServiceModules, extensionModules);
+ }
+
+ protected void updateArchives(
+ File resourcesRoot, File stackRoot, Map<String, StackModule> stackModules, Map<String, ServiceModule> commonServiceModules,
+ Map<String, ExtensionModule> extensionModules ) throws AmbariException {
+
+ LOG.info("Refreshing archives ...");
+
+ LOG.debug("Refreshing archives for stacks");
+ for (StackModule stackModule : stackModules.values()) {
+ LOG.debug("Refreshing archives for stack : " + stackModule.getId());
- String hooksDir = stackModule.getStackDirectory().getHooksDir();
- if(hooksDir != null) {
- LOG.debug("Refreshing archive for stack hooks directory : " + hooksDir);
- String hooksAbsolutePath = stackRoot.getAbsolutePath() + File.separator + hooksDir;
- ResourceFilesKeeperHelper.updateDirectoryArchive(hooksAbsolutePath, false);
- }
+ for(ServiceModule serviceModule : stackModule.getServiceModules().values()) {
+ String packageDir = serviceModule.getServiceDirectory().getPackageDir();
+ if(packageDir != null) {
+ LOG.debug("Refreshing archive for stack service package directory : " + packageDir);
+ String packageAbsoluteDir = resourcesRoot.getAbsolutePath() + File.separator + packageDir;
+ ResourceFilesKeeperHelper.updateDirectoryArchive(packageAbsoluteDir, false);
+ }
+ }
+ }
+
+ LOG.debug("Refreshing archives for common services");
+ for(ServiceModule serviceModule : commonServiceModules.values()) {
+ String packageDir = serviceModule.getServiceDirectory().getPackageDir();
+ if(packageDir != null) {
+ LOG.debug("Refreshing archive for common service package directory : " + packageDir);
+ String packageAbsoluteDir = resourcesRoot.getAbsolutePath() + File.separator + packageDir;
+ ResourceFilesKeeperHelper.updateDirectoryArchive(packageAbsoluteDir, false);
+ }
+ }
+
+ LOG.debug("Refreshing archives for extensions");
+ for(ExtensionModule extensionModule : extensionModules.values()) {
+ LOG.debug("Refreshing archives for extension module" + extensionModule.getId());
+ for(ServiceModule serviceModule : extensionModule.getServiceModules().values()) {
+ String packageDir = serviceModule.getServiceDirectory().getPackageDir();
+ if(packageDir != null) {
+ LOG.debug("Refreshing archive for extension service package directory : " + packageDir);
+ String packageAbsoluteDir = resourcesRoot.getAbsolutePath() + File.separator + packageDir;
+ ResourceFilesKeeperHelper.updateDirectoryArchive(packageAbsoluteDir, false);
+ }
+ }
+ }
+
+ List<String> miscDirs = new ArrayList<String>() {{
+ add(CUSTOM_ACTIONS_DIR);
+ add(HOST_SCRIPTS_DIR);
+ add(DASHBOARDS_DIR);
+ }};
+
+ LOG.debug("Refreshing archives for misc directories");
+ for(String miscDir : miscDirs) {
+ LOG.debug("Refreshing archive for misc directory : " + miscDir);
+ String miscAbsolutePath = resourcesRoot.getAbsolutePath() + File.separator + miscDir;
+ ResourceFilesKeeperHelper.updateDirectoryArchive(miscAbsolutePath, false);
+ }
+ LOG.info("Refreshing archives finished!");
+ }
+
protected void parseDirectories(File stackRoot, File commonServicesRoot, File extensionRoot) throws AmbariException {
commonServiceModules = parseCommonServicesDirectory(commonServicesRoot);
stackModules = parseStackDirectory(stackRoot);
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
index 56d352e,4af83ef..13ceeb9
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
@@@ -64,7 -67,8 +67,9 @@@ import org.apache.ambari.server.state.H
import org.apache.ambari.server.state.HostHealthStatus.HealthStatus;
import org.apache.ambari.server.state.HostState;
import org.apache.ambari.server.state.MaintenanceState;
++import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.StackId;
+ import org.apache.ambari.server.state.UpgradeState;
import org.apache.ambari.server.state.configgroup.ConfigGroup;
import org.apache.ambari.server.state.fsm.InvalidStateTransitionException;
import org.apache.ambari.server.state.fsm.SingleArcTransition;
@@@ -1168,6 -1172,45 +1173,48 @@@ public class HostImpl implements Host
return false;
}
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean isRepositoryVersionCorrect(RepositoryVersionEntity repositoryVersion)
+ throws AmbariException {
+ HostEntity hostEntity = getHostEntity();
+ Collection<HostComponentStateEntity> hostComponentStates = hostEntity.getHostComponentStateEntities();
+
+ // for every host component, if it matches the desired repo and has reported
+ // the correct version then we're good
+ for (HostComponentStateEntity hostComponentState : hostComponentStates) {
+ ServiceComponentDesiredStateEntity desiredComponmentState = hostComponentState.getServiceComponentDesiredStateEntity();
+ RepositoryVersionEntity desiredRepositoryVersion = desiredComponmentState.getDesiredRepositoryVersion();
+
++ Long clusterId = hostComponentState.getClusterId();
++ Cluster cluster = clusters.getCluster(clusterId);
++ Service s = cluster.getService(hostComponentState.getServiceId());
+ ComponentInfo componentInfo = ambariMetaInfo.getComponent(
+ desiredRepositoryVersion.getStackName(), desiredRepositoryVersion.getStackVersion(),
- hostComponentState.getServiceName(), hostComponentState.getComponentName());
++ s.getName(), hostComponentState.getComponentName());
+
+ // skip components which don't advertise a version
+ if (!componentInfo.isVersionAdvertised()) {
+ continue;
+ }
+
+ // we only care about checking the specified repo version for this host
+ if (!repositoryVersion.equals(desiredRepositoryVersion)) {
+ continue;
+ }
+
+ String versionAdvertised = hostComponentState.getVersion();
+ if (hostComponentState.getUpgradeState() == UpgradeState.IN_PROGRESS
+ || !StringUtils.equals(versionAdvertised, repositoryVersion.getVersion())) {
+ return false;
+ }
+ }
+
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.3.0/service_advisor.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/resources/common-services/RANGER_KMS/1.0.0.3.0/service_advisor.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
----------------------------------------------------------------------
diff --cc ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
index dbe2eed,9285526..8c44632
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
@@@ -73,6 -71,6 +73,7 @@@ import org.apache.ambari.server.state.O
import org.apache.ambari.server.state.PropertyDependencyInfo;
import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.RepositoryInfo;
++import org.apache.ambari.server.state.ServiceGroup;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.StackInfo;
@@@ -1738,8 -1716,8 +1721,8 @@@ public class AmbariMetaInfoTest
RepositoryVersionEntity repositoryVersion = ormHelper.getOrCreateRepositoryVersion(
cluster.getCurrentStackVersion(), repoVersion);
--
-- cluster.addService("HDFS", repositoryVersion);
++ ServiceGroup sg = cluster.addServiceGroup("core");
++ cluster.addService(sg, "HDFS", "HDFS", repositoryVersion);
metaInfo.reconcileAlertDefinitions(clusters);
@@@ -1962,7 -1947,7 +1952,8 @@@
Properties properties = new Properties();
properties.setProperty(Configuration.METADATA_DIR_PATH.getKey(), stackRoot.getPath());
properties.setProperty(Configuration.SERVER_VERSION_FILE.getKey(), versionFile.getPath());
+ properties.setProperty(Configuration.MPACKS_V2_STAGING_DIR_PATH.getKey(),"src/test/resources/mpacks-v2");
+ properties.setProperty(Configuration.RESOURCES_DIR.getKey(), resourcesRoot.getPath());
Configuration configuration = new Configuration(properties);
TestAmbariMetaInfo metaInfo = new TestAmbariMetaInfo(configuration);
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelperTest.java
----------------------------------------------------------------------
diff --cc ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelperTest.java
index f7f7ac6,a3d13b1..049fe27
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorUpdateHelperTest.java
@@@ -69,11 -66,10 +69,12 @@@ public class KerberosDescriptorUpdateHe
properties.put("common.services.path", "src/main/resources/common-services");
properties.put("server.version.file", "target/version");
properties.put("custom.action.definitions", "/tmp/nofile");
+ properties.put("mpacks-v2.staging.path","src/test/resources/mpacks-v2");
+ properties.put("resources.dir", "src/main/resources");
Configuration configuration = new Configuration(properties);
- install(new FactoryModuleBuilder().build(StackManagerFactory.class));
+ install(new FactoryModuleBuilder().implement(
+ StackManager.class, StackManagerMock.class).build(StackManagerFactory.class));
bind(Clusters.class).toInstance(createNiceMock(Clusters.class));
bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-web/app/controllers/installer.js
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-web/app/controllers/wizard/step8_controller.js
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-web/app/utils/ajax/ajax.js
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-web/test/controllers/installer_test.js
----------------------------------------------------------------------
diff --cc ambari-web/test/controllers/installer_test.js
index 37c2303,65a1e5c..d2d4f2c
--- a/ambari-web/test/controllers/installer_test.js
+++ b/ambari-web/test/controllers/installer_test.js
@@@ -494,27 -468,44 +494,44 @@@ describe('App.InstallerController', fun
};
beforeEach(function () {
+ sinon.spy(checker, 'loadStacks');
- installerController.loadMap['1'][0].callback.call(checker);
+ installerController.loadMap['step1'][0].callback.call(checker);
});
- it('stack info is loaded', function () {
- expect(loadStacks).to.be.true;
+ afterEach(function() {
+ checker.loadStacks.restore();
+ });
+
+ it('should call loadStacks, stack info not loaded', function () {
+ expect(checker.loadStacks.calledOnce).to.be.true;
});
});
- describe ('Should load stacks async', function() {
- var loadStacksVersions = false;
+ describe('Should load stacks async', function() {
var checker = {
- loadStacksVersions: function() {
- loadStacksVersions = true;
- }
+ loadStacksVersions: Em.K
};
+ beforeEach(function () {
+ sinon.spy(checker, 'loadStacksVersions');
+ });
+
+ afterEach(function() {
+ checker.loadStacksVersions.restore();
+ });
+
it('stack versions are loaded', function () {
- installerController.loadMap['1'][1].callback.call(checker, true).then(function(data){
+ installerController.loadMap['step1'][1].callback.call(checker, true).then(function(data){
expect(data).to.be.true;
});
- expect(loadStacksVersions).to.be.false;
+ expect(checker.loadStacksVersions.called).to.be.false;
+ });
+
+ it('should call loadStacksVersions, stack versions not loaded', function () {
+ installerController.loadMap['1'][1].callback.call(checker, false).then(function(data){
+ expect(data).to.be.true;
+ });
+ expect(checker.loadStacksVersions.calledOnce).to.be.true;
});
});
http://git-wip-us.apache.org/repos/asf/ambari/blob/c36afcdd/ambari-web/test/mixins/common/configs/configs_saver_test.js
----------------------------------------------------------------------