You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2018/02/07 13:45:17 UTC
[ambari] branch branch-feature-AMBARI-14714 updated: Merge trunk
into branch-feature-AMBARI-14714
This is an automated email from the ASF dual-hosted git repository.
dsen pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push:
new 260876c Merge trunk into branch-feature-AMBARI-14714
260876c is described below
commit 260876ce6e8c3b1699ef61b962f87097d85da444
Author: Dmytro Sen <ds...@apache.org>
AuthorDate: Wed Feb 7 15:39:54 2018 +0200
Merge trunk into branch-feature-AMBARI-14714
---
.../main/resources/ui/admin-web/app/scripts/app.js | 2 +-
.../src/main/resources/ui/admin-web/package.json | 2 +-
ambari-agent/conf/unix/ambari-agent | 8 +-
ambari-agent/conf/unix/ambari-env.sh | 2 +-
ambari-agent/conf/unix/install-helper.sh | 57 +-
ambari-agent/pom.xml | 13 +-
.../src/main/package/rpm/posttrans_agent.sh | 20 -
.../src/main/python/ambari_agent/ActionQueue.py | 9 +-
.../src/main/python/ambari_agent/AmbariAgent.py | 2 +-
.../src/main/python/ambari_agent/HostCleanup.py | 2 +-
.../test/python/ambari_agent/TestProcessUtils.py | 224 -
.../src/test/python/ambari_agent/TestShell.py | 177 +-
.../main/python/ambari_commons/process_utils.py | 100 -
.../src/main/python/ambari_commons/shell.py | 182 +-
.../libraries/functions/lzo_utils.py | 9 +-
.../libraries/functions/namenode_ha_utils.py | 14 +-
.../libraries/functions/repository_util.py | 99 +-
.../libraries/functions/solr_cloud_util.py | 46 +-
.../libraries/functions/tar_archive.py | 4 +-
.../resource_management/libraries/script/script.py | 10 +-
.../org/apache/ambari/infra/HttpResponse.java} | 20 +-
.../java/org/apache/ambari/infra/InfraClient.java | 32 +-
.../org/apache/ambari/infra/JobExecutionInfo.java | 35 +-
.../ambari/infra/steps/AbstractInfraSteps.java | 12 +-
.../apache/ambari/infra/steps/ExportJobsSteps.java | 66 +-
.../test/resources/stories/infra_api_tests.story | 26 +-
.../ambari-infra-manager/docs/api/swagger.yaml | 2 +-
ambari-infra/ambari-infra-manager/pom.xml | 1 -
.../InfraManagerSchedulingConfig.java} | 15 +-
.../infra/conf/batch/InfraManagerBatchConfig.java | 25 +-
.../conf/security/CompositePasswordStore.java | 31 +-
.../infra/conf/security/HadoopCredentialStore.java | 55 +
.../conf/security/InfraManagerSecurityConfig.java | 30 +-
.../security/PasswordStore.java} | 8 +-
.../security/SecurityEnvironment.java} | 11 +-
.../infra/job/AbstractJobsConfiguration.java | 82 +
...ertyMap.java => JobConfigurationException.java} | 8 +-
...{PropertyMap.java => JobContextRepository.java} | 7 +-
.../ambari/infra/job/JobContextRepositoryImpl.java | 52 +
.../org/apache/ambari/infra/job/JobProperties.java | 35 +
.../org/apache/ambari/infra/job/JobScheduler.java | 89 +
.../{JobPropertyMap.java => JobsPropertyMap.java} | 12 +-
...{PropertyMap.java => SchedulingProperties.java} | 30 +-
.../archive/DocumentArchivingConfiguration.java | 65 +-
.../job/archive/DocumentArchivingProperties.java | 55 +-
.../job/archive/DocumentArchivingPropertyMap.java | 8 +-
.../ambari/infra/job/archive/DocumentExporter.java | 22 +-
.../infra/job/archive/FileNameSuffixFormatter.java | 2 +-
.../ambari/infra/job/archive/S3AccessCsv.java | 94 +
.../S3AccessCsvFormatException.java} | 10 +-
.../ambari/infra/job/archive/S3AccessKeyNames.java | 32 +-
.../ambari/infra/job/archive/S3Properties.java | 28 +-
.../ambari/infra/job/archive/S3Uploader.java | 16 +-
.../deleting/DocumentDeletingConfiguration.java | 53 +-
.../job/deleting/DocumentDeletingPropertyMap.java | 8 +-
.../apache/ambari/infra/manager/JobManager.java | 64 +-
.../java/org/apache/ambari/infra/manager/Jobs.java | 42 +
.../org/apache/ambari/infra/rest/JobResource.java | 28 +-
.../src/main/resources/infra-manager.properties | 8 +-
.../src/main/resources/log4j2.xml | 3 +
.../conf/security/CompositePasswordStoreTest.java | 48 +
.../apache/ambari/infra/job/JobSchedulerTest.java | 114 +
.../infra/job/archive/DocumentExporterTest.java | 110 +-
.../ambari/infra/job/archive/S3AccessCsvTest.java | 70 +
.../vagrant-infra-manager.properties.sample | 30 +-
ambari-infra/ambari-infra-solr-client/pom.xml | 26 +-
.../src/main/resources/solrCloudCli.sh | 2 +-
ambari-infra/pom.xml | 100 +-
ambari-logsearch/ambari-logsearch-appender/pom.xml | 23 +
.../ambari-logsearch-logfeeder/pom.xml | 1 +
.../ambari/logfeeder/output/OutputManagerImpl.java | 9 +-
.../apache/ambari/logfeeder/util/MurmurHash.java | 163 -
ambari-logsearch/ambari-logsearch-server/pom.xml | 2 +
.../logsearch/auth/filter/AbstractJWTFilter.java | 37 +-
.../logsearch/common/LogSearchConstants.java | 16 +-
.../ambari/logsearch/common/MessageEnums.java | 2 +-
.../ambari/logsearch/conf/AuthPropsConfig.java | 20 +-
.../logsearch/conf/LogSearchConfigApiConfig.java | 37 +-
.../ambari/logsearch/conf/LogSearchSslConfig.java | 2 +-
.../ambari/logsearch/conf/SecurityConfig.java | 24 +-
.../configurer/LogSearchConfigConfigurer.java | 12 +-
.../AbstractOperationHolderConverter.java | 13 +-
.../AuditBarGraphRequestQueryConverter.java | 8 +
.../AuditComponentsRequestQueryConverter.java | 11 +
.../converter/AuditLogRequestQueryConverter.java | 7 +
.../AuditServiceLoadRequestQueryConverter.java | 10 +
.../FieldAuditLogRequestQueryConverter.java | 9 +
.../TopFieldAuditLogRequestQueryConverter.java | 9 +
.../converter/UserExportRequestQueryConverter.java | 10 +
.../apache/ambari/logsearch/dao/SolrDaoBase.java | 16 +-
.../apache/ambari/logsearch/doc/DocConstants.java | 2 +
.../logsearch/handler/CreateCollectionHandler.java | 2 +-
.../ambari/logsearch/manager/InfoManager.java | 13 +-
.../logsearch/manager/ServiceLogsManager.java | 2 +-
.../logsearch/manager/ShipperConfigManager.java | 13 +-
...ogRequest.java => ShipperConfigTestParams.java} | 33 +-
...ceLoadRequest.java => UserParamDefinition.java} | 16 +-
.../model/request/impl/AuditBarGraphRequest.java | 16 +-
.../model/request/impl/AuditComponentRequest.java | 17 +-
.../model/request/impl/AuditLogRequest.java | 16 +-
.../request/impl/AuditServiceLoadRequest.java | 19 +-
.../model/request/impl/CommonSearchRequest.java | 3 +
.../request/impl/FieldAuditBarGraphRequest.java | 16 +-
.../model/request/impl/FieldAuditLogRequest.java | 16 +-
...hRequest.java => ShipperConfigTestRequest.java} | 45 +-
.../request/impl/TopFieldAuditLogRequest.java | 17 +-
.../model/request/impl/UserExportRequest.java | 16 +-
.../apache/ambari/logsearch/rest/InfoResource.java | 25 +-
.../logsearch/rest/ShipperConfigResource.java | 8 +-
.../web/filters/LogSearchConfigStateFilter.java | 13 +-
.../logsearch/web/filters/LogsearchJWTFilter.java | 5 +
.../AuditBarGraphRequestQueryConverterTest.java | 3 +-
.../converter/AuditLogRequestConverterTest.java | 3 +-
.../ambari-logsearch-web/src/app/app.module.ts | 19 +-
.../classes/components/graph/graph.component.ts | 18 +-
.../src/app/classes/filtering.ts | 2 +-
.../src/app/classes/list-item.ts | 5 +-
.../src/app/classes/models/app-settings.ts | 8 +-
.../src/app/classes/models/app-state.ts | 13 +-
.../src/app/classes/models/filter.ts | 8 +-
.../src/app/classes/models/store.ts | 45 +-
.../ambari-logsearch-web/src/app/classes/object.ts | 8 +
.../app/classes/queries/audit-logs-query-params.ts | 33 +-
.../audit-logs-top-resources-query-params.ts | 4 +-
...t-logs-query-params.ts => logs-query-params.ts} | 12 +-
.../classes/queries/service-logs-query-params.ts | 4 +-
.../queries/service-logs-truncated-query-params.ts | 2 +-
.../app/classes/{models/filter.ts => settings.ts} | 17 +-
.../ambari-logsearch-web/src/app/classes/string.ts | 2 +
.../action-menu/action-menu.component.html | 31 +-
.../action-menu/action-menu.component.less | 5 +
.../action-menu/action-menu.component.spec.ts | 82 +-
.../action-menu/action-menu.component.ts | 149 +-
.../audit-logs-entries.component.html | 4 +-
.../audit-logs-entries.component.spec.ts | 2 +
.../audit-logs-table.component.html | 7 +-
.../audit-logs-table/audit-logs-table.component.ts | 8 +-
.../context-menu/context-menu.component.spec.ts | 6 +-
.../date-picker/date-picker.component.ts | 21 +-
.../dropdown-button/dropdown-button.component.html | 20 +-
.../dropdown-button/dropdown-button.component.less | 8 +-
.../dropdown-button.component.spec.ts | 12 +-
.../dropdown-button/dropdown-button.component.ts | 40 +-
.../dropdown-list/dropdown-list.component.html | 28 +-
.../dropdown-list/dropdown-list.component.spec.ts | 6 +-
.../dropdown-list/dropdown-list.component.ts | 39 +-
.../filter-button/filter-button.component.spec.ts | 12 +-
.../filter-dropdown.component.spec.ts | 12 +-
.../filter-dropdown/filter-dropdown.component.ts | 5 -
.../filters-panel/filters-panel.component.html | 13 +-
.../filters-panel/filters-panel.component.less | 4 -
.../filters-panel/filters-panel.component.spec.ts | 4 +-
.../filters-panel/filters-panel.component.ts | 13 +-
.../history-item-controls.component.html} | 8 +-
.../history-item-controls.component.less} | 4 +-
.../history-item-controls.component.spec.ts} | 16 +-
.../history-item-controls.component.ts} | 11 +-
.../log-context/log-context.component.spec.ts | 4 +-
.../log-index-filter.component.html | 86 +
.../log-index-filter.component.less} | 40 +-
.../log-index-filter.component.spec.ts} | 61 +-
.../log-index-filter/log-index-filter.component.ts | 177 +
.../logs-container/logs-container.component.ts | 20 +-
.../menu-button/menu-button.component.html | 8 +-
.../menu-button/menu-button.component.spec.ts | 12 +-
.../menu-button/menu-button.component.ts | 28 +-
.../src/app/components/modal/modal.component.html | 5 +-
.../modal.component.less} | 10 +-
.../src/app/components/modal/modal.component.ts | 9 +-
.../pagination/pagination.component.html | 3 +-
.../components/search-box/search-box.component.ts | 4 +-
.../service-logs-table.component.html | 13 +-
.../service-logs-table.component.less | 2 +-
.../service-logs-table.component.spec.ts | 2 -
.../service-logs-table.component.ts | 99 +-
.../time-range-picker.component.spec.ts | 2 +
.../time-range-picker.component.ts | 2 +-
.../timezone-picker.component.spec.ts | 8 +-
.../timezone-picker/timezone-picker.component.ts | 15 +-
.../components/top-menu/top-menu.component.html | 8 +-
.../components/top-menu/top-menu.component.spec.ts | 4 +
.../app/components/top-menu/top-menu.component.ts | 23 +-
.../src/app/components/variables.less | 5 +-
.../ambari-logsearch-web/src/app/mock-data.ts | 1500 +++-
.../src/app/services/component-actions.service.ts | 155 -
.../services/component-generator.service.spec.ts | 2 +
.../app/services/component-generator.service.ts | 15 +-
...ice.spec.ts => history-manager.service.spec.ts} | 94 +-
.../src/app/services/history-manager.service.ts | 330 +
.../src/app/services/http-client.service.ts | 9 +-
.../app/services/logs-container.service.spec.ts | 30 +-
.../src/app/services/logs-container.service.ts | 249 +-
.../src/app/services/mock-api-data.service.ts | 55 +-
.../src/app/services/storage/reducers.service.ts | 2 -
...rvice.spec.ts => user-settings.service.spec.ts} | 75 +-
.../src/app/services/user-settings.service.ts | 162 +
.../src/app/services/utils.service.spec.ts | 149 +
.../src/app/services/utils.service.ts | 46 +
.../ambari-logsearch-web/src/assets/i18n/en.json | 19 +-
ambari-logsearch/docker/Dockerfile | 19 +
ambari-logsearch/docker/bin/start.sh | 56 +-
.../docker/knox.yml | 38 +-
.../docker/knox/gateway.sh | 9 +-
.../docker/knox/keystores/test-secrets.zip | Bin 0 -> 3841 bytes
.../docker/knox/ldap.sh | 9 +-
.../docker/knox/logsearch/1.0.0/rewrite.xml | 23 +
.../docker/knox/logsearch/1.0.0/service.xml | 12 +
ambari-logsearch/docker/knox/topologies/admin.xml | 67 +
.../docker/knox/topologies/knoxsso.xml | 118 +
.../docker/knox/topologies/sandbox.xml | 55 +
ambari-logsearch/docker/logsearch-docker.sh | 1 +
ambari-logsearch/docker/sso.yml | 119 +
.../logsearch/logsearch-https.properties | 2 +
...h-https.properties => logsearch-sso.properties} | 10 +-
ambari-logsearch/pom.xml | 92 +-
ambari-project/pom.xml | 6 +
ambari-server/conf/ambari-env.sh | 1 +
ambari-server/conf/unix/ambari-env.sh | 2 +-
ambari-server/conf/unix/ambari.properties | 4 +-
ambari-server/conf/unix/install-helper.sh | 43 +-
ambari-server/docs/configuration/index.md | 32 +-
ambari-server/pom.xml | 29 +-
ambari-server/src/main/assemblies/server.xml | 8 +-
.../actionmanager/ExecutionCommandWrapper.java | 4 +-
.../ambari/server/agent/CommandRepository.java | 22 +-
.../ambari/server/configuration/Configuration.java | 21 +-
.../controller/AmbariManagementControllerImpl.java | 20 +-
.../ambari/server/controller/AmbariServer.java | 4 +
.../ambari/server/controller/KerberosHelper.java | 11 +
.../server/controller/KerberosHelperImpl.java | 2 +-
.../internal/AmbariServerConfigurationHandler.java | 9 +-
.../ClusterStackVersionResourceProvider.java | 20 +-
.../internal/HostStackVersionResourceProvider.java | 10 +-
.../RepositoryVersionResourceProvider.java | 23 +-
.../controller/internal/UserResourceProvider.java | 19 +-
.../VersionDefinitionResourceProvider.java | 20 +-
.../controller/logging/HostLogFilesResponse.java | 27 +-
.../logging/LogSearchDataRetrievalService.java | 21 +-
.../controller/logging/LoggingRequestHelper.java | 11 +-
.../logging/LoggingRequestHelperFactoryImpl.java | 72 +-
.../logging/LoggingRequestHelperImpl.java | 111 +-
.../service/AmbariLdapConfigurationProvider.java | 12 +-
.../server/orm/dao/RepositoryVersionDAO.java | 17 +-
.../server/orm/entities/OperatingSystemEntity.java | 78 -
.../server/orm/entities/RepoDefinitionEntity.java | 212 +
.../ambari/server/orm/entities/RepoOsEntity.java | 164 +
.../server/orm/entities/RepositoryEntity.java | 135 -
.../orm/entities/RepositoryVersionEntity.java | 77 +-
.../AmbariLdapAuthenticationProvider.java | 16 +-
.../security/ldap/AmbariLdapDataPopulator.java | 28 +-
.../AbstractPrepareKerberosServerAction.java | 2 +-
.../server/stack/ClassifyNameNodeException.java | 26 +
.../org/apache/ambari/server/stack/HostsType.java | 152 +-
.../ambari/server/stack/MasterHostResolver.java | 209 +-
.../apache/ambari/server/stack/NameService.java | 157 +
.../org/apache/ambari/server/stack/RepoUtil.java | 33 +-
.../stack/UpdateActiveRepoVersionOnStartup.java | 7 +-
.../apache/ambari/server/state/UpgradeHelper.java | 49 +-
.../state/stack/upgrade/ClusterGrouping.java | 18 +-
.../state/stack/upgrade/ColocatedGrouping.java | 7 +-
.../server/state/stack/upgrade/Grouping.java | 2 +-
.../state/stack/upgrade/HostOrderGrouping.java | 2 +-
.../stack/upgrade/RepositoryVersionHelper.java | 155 +-
.../state/stack/upgrade/TaskWrapperBuilder.java | 16 +-
.../svccomphost/ServiceComponentHostImpl.java | 2 +-
.../server/topology/AsyncCallableService.java | 16 +-
.../topology/tasks/ConfigureClusterTask.java | 5 +-
.../server/upgrade/AbstractUpgradeCatalog.java | 22 +
.../ambari/server/upgrade/UpgradeCatalog300.java | 396 +-
.../src/main/package/rpm/posttrans_server.sh | 20 -
ambari-server/src/main/python/ambari-server.py | 25 +-
.../python/ambari_server/serverConfiguration.py | 7 +-
.../src/main/python/ambari_server/serverUpgrade.py | 8 +-
.../src/main/python/ambari_server/setupSecurity.py | 219 +-
ambari-server/src/main/python/bootstrap.py | 4 +-
.../src/main/resources/Ambari-DDL-Derby-CREATE.sql | 30 +-
.../src/main/resources/Ambari-DDL-MySQL-CREATE.sql | 28 +-
.../main/resources/Ambari-DDL-Oracle-CREATE.sql | 30 +-
.../main/resources/Ambari-DDL-Postgres-CREATE.sql | 31 +-
.../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 29 +-
.../main/resources/Ambari-DDL-SQLServer-CREATE.sql | 29 +-
.../src/main/resources/META-INF/persistence.xml | 2 +
.../0.1.0/configuration/infra-solr-env.xml | 29 +-
.../0.1.0/package/scripts/infra_solr.py | 2 +-
.../AMBARI_INFRA/0.1.0/package/scripts/params.py | 27 +-
.../0.1.0/package/scripts/setup_infra_solr.py | 37 +-
.../0.1.0/properties/infra-solr-env.sh.j2 | 1 +
.../common-services/ATLAS/0.7.0.3.0/alerts.json | 39 -
.../configuration/application-properties.xml | 546 --
.../ATLAS/0.7.0.3.0/configuration/atlas-env.xml | 200 -
.../ATLAS/0.7.0.3.0/configuration/atlas-log4j.xml | 170 -
.../0.7.0.3.0/configuration/atlas-solrconfig.xml | 641 --
.../0.7.0.3.0/configuration/ranger-atlas-audit.xml | 141 -
.../ranger-atlas-plugin-properties.xml | 132 -
.../configuration/ranger-atlas-policymgr-ssl.xml | 73 -
.../configuration/ranger-atlas-security.xml | 77 -
.../common-services/ATLAS/0.7.0.3.0/kerberos.json | 103 -
.../common-services/ATLAS/0.7.0.3.0/metainfo.xml | 193 -
.../0.7.0.3.0/package/scripts/atlas_client.py | 53 -
.../ATLAS/0.7.0.3.0/package/scripts/metadata.py | 243 -
.../0.7.0.3.0/package/scripts/metadata_server.py | 183 -
.../ATLAS/0.7.0.3.0/package/scripts/params.py | 419 --
.../0.7.0.3.0/package/scripts/service_check.py | 55 -
.../package/scripts/setup_ranger_atlas.py | 70 -
.../0.7.0.3.0/package/scripts/status_params.py | 60 -
.../package/templates/atlas_hbase_setup.rb.j2 | 42 -
.../0.7.0.3.0/package/templates/atlas_jaas.conf.j2 | 26 -
.../package/templates/atlas_kafka_acl.sh.j2 | 41 -
.../package/templates/input.config-atlas.json.j2 | 48 -
.../0.7.0.3.0/package/templates/kafka_jaas.conf.j2 | 41 -
.../ATLAS/0.7.0.3.0/quicklinks/quicklinks.json | 36 -
.../ATLAS/0.7.0.3.0/role_command_order.json | 7 -
.../ATLAS/0.7.0.3.0/service_advisor.py | 444 --
.../ATLAS/0.7.0.3.0/themes/theme.json | 619 --
.../ATLAS/0.7.0.3.0/themes/theme_version_2.json | 845 ---
.../common-services/HBASE/2.0.0.3.0/alerts.json | 127 -
.../HBASE/2.0.0.3.0/configuration/hbase-env.xml | 285 -
.../HBASE/2.0.0.3.0/configuration/hbase-log4j.xml | 188 -
.../HBASE/2.0.0.3.0/configuration/hbase-policy.xml | 53 -
.../HBASE/2.0.0.3.0/configuration/hbase-site.xml | 774 --
.../2.0.0.3.0/configuration/ranger-hbase-audit.xml | 132 -
.../ranger-hbase-plugin-properties.xml | 135 -
.../configuration/ranger-hbase-policymgr-ssl.xml | 66 -
.../configuration/ranger-hbase-security.xml | 74 -
.../common-services/HBASE/2.0.0.3.0/kerberos.json | 165 -
.../common-services/HBASE/2.0.0.3.0/metainfo.xml | 238 -
.../common-services/HBASE/2.0.0.3.0/metrics.json | 4733 ------------
.../2.0.0.3.0/package/files/draining_servers.rb | 164 -
.../2.0.0.3.0/package/files/hbase-smoke-cleanup.sh | 23 -
.../2.0.0.3.0/package/files/hbaseSmokeVerify.sh | 34 -
.../HBASE/2.0.0.3.0/package/scripts/__init__.py | 19 -
.../HBASE/2.0.0.3.0/package/scripts/functions.py | 54 -
.../HBASE/2.0.0.3.0/package/scripts/hbase.py | 239 -
.../2.0.0.3.0/package/scripts/hbase_client.py | 69 -
.../package/scripts/hbase_decommission.py | 94 -
.../2.0.0.3.0/package/scripts/hbase_master.py | 160 -
.../package/scripts/hbase_regionserver.py | 171 -
.../2.0.0.3.0/package/scripts/hbase_service.py | 66 -
.../2.0.0.3.0/package/scripts/hbase_upgrade.py | 42 -
.../HBASE/2.0.0.3.0/package/scripts/params.py | 28 -
.../2.0.0.3.0/package/scripts/params_linux.py | 426 --
.../2.0.0.3.0/package/scripts/params_windows.py | 43 -
.../package/scripts/phoenix_queryserver.py | 86 -
.../2.0.0.3.0/package/scripts/phoenix_service.py | 56 -
.../2.0.0.3.0/package/scripts/service_check.py | 99 -
.../package/scripts/setup_ranger_hbase.py | 106 -
.../2.0.0.3.0/package/scripts/status_params.py | 68 -
.../HBASE/2.0.0.3.0/package/scripts/upgrade.py | 105 -
.../2.0.0.3.0/package/templates/hbase-smoke.sh.j2 | 44 -
.../2.0.0.3.0/package/templates/hbase.conf.j2 | 35 -
.../package/templates/hbase_client_jaas.conf.j2 | 23 -
.../package/templates/hbase_grant_permissions.j2 | 39 -
.../package/templates/hbase_master_jaas.conf.j2 | 36 -
.../templates/hbase_queryserver_jaas.conf.j2 | 26 -
.../templates/hbase_regionserver_jaas.conf.j2 | 36 -
.../package/templates/input.config-hbase.json.j2 | 79 -
.../2.0.0.3.0/package/templates/regionservers.j2 | 20 -
.../HBASE/2.0.0.3.0/quicklinks/quicklinks.json | 97 -
.../HBASE/2.0.0.3.0/role_command_order.json | 10 -
.../HBASE/2.0.0.3.0/service_advisor.py | 685 --
.../HBASE/2.0.0.3.0/themes/theme.json | 407 -
.../common-services/HBASE/2.0.0.3.0/widgets.json | 510 --
.../HDFS/2.1.0.2.0/configuration/hadoop-env.xml | 1 +
.../common-services/HDFS/3.0.0.3.0/alerts.json | 1786 -----
.../HDFS/3.0.0.3.0/configuration/core-site.xml | 224 -
.../HDFS/3.0.0.3.0/configuration/hadoop-env.xml | 436 --
.../configuration/hadoop-metrics2.properties.xml | 127 -
.../HDFS/3.0.0.3.0/configuration/hadoop-policy.xml | 130 -
.../HDFS/3.0.0.3.0/configuration/hdfs-log4j.xml | 227 -
.../configuration/hdfs-logsearch-conf.xml | 248 -
.../HDFS/3.0.0.3.0/configuration/hdfs-site.xml | 647 --
.../3.0.0.3.0/configuration/ranger-hdfs-audit.xml | 132 -
.../ranger-hdfs-plugin-properties.xml | 96 -
.../configuration/ranger-hdfs-policymgr-ssl.xml | 67 -
.../configuration/ranger-hdfs-security.xml | 65 -
.../HDFS/3.0.0.3.0/configuration/ssl-client.xml | 70 -
.../HDFS/3.0.0.3.0/configuration/ssl-server.xml | 80 -
.../common-services/HDFS/3.0.0.3.0/kerberos.json | 254 -
.../common-services/HDFS/3.0.0.3.0/metainfo.xml | 376 -
.../common-services/HDFS/3.0.0.3.0/metrics.json | 7905 --------------------
.../package/alerts/alert_checkpoint_time.py | 255 -
.../alerts/alert_datanode_unmounted_data_dir.py | 177 -
.../package/alerts/alert_ha_namenode_health.py | 243 -
.../package/alerts/alert_metrics_deviation.py | 478 --
.../package/alerts/alert_upgrade_finalized.py | 179 -
.../HDFS/3.0.0.3.0/package/files/checkWebUI.py | 83 -
.../HDFS/3.0.0.3.0/package/scripts/__init__.py | 20 -
.../scripts/balancer-emulator/balancer-err.log | 1032 ---
.../package/scripts/balancer-emulator/balancer.log | 29 -
.../scripts/balancer-emulator/hdfs-command.py | 45 -
.../HDFS/3.0.0.3.0/package/scripts/datanode.py | 126 -
.../3.0.0.3.0/package/scripts/datanode_upgrade.py | 156 -
.../HDFS/3.0.0.3.0/package/scripts/hdfs.py | 251 -
.../HDFS/3.0.0.3.0/package/scripts/hdfs_client.py | 78 -
.../3.0.0.3.0/package/scripts/hdfs_datanode.py | 85 -
.../3.0.0.3.0/package/scripts/hdfs_namenode.py | 613 --
.../3.0.0.3.0/package/scripts/hdfs_nfsgateway.py | 75 -
.../3.0.0.3.0/package/scripts/hdfs_rebalance.py | 156 -
.../3.0.0.3.0/package/scripts/hdfs_snamenode.py | 72 -
.../3.0.0.3.0/package/scripts/install_params.py | 29 -
.../HDFS/3.0.0.3.0/package/scripts/journalnode.py | 143 -
.../package/scripts/journalnode_upgrade.py | 152 -
.../HDFS/3.0.0.3.0/package/scripts/namenode.py | 381 -
.../3.0.0.3.0/package/scripts/namenode_ha_state.py | 219 -
.../3.0.0.3.0/package/scripts/namenode_upgrade.py | 322 -
.../HDFS/3.0.0.3.0/package/scripts/nfsgateway.py | 88 -
.../HDFS/3.0.0.3.0/package/scripts/params.py | 28 -
.../HDFS/3.0.0.3.0/package/scripts/params_linux.py | 551 --
.../3.0.0.3.0/package/scripts/params_windows.py | 86 -
.../3.0.0.3.0/package/scripts/service_check.py | 158 -
.../3.0.0.3.0/package/scripts/setup_ranger_hdfs.py | 120 -
.../HDFS/3.0.0.3.0/package/scripts/snamenode.py | 101 -
.../3.0.0.3.0/package/scripts/status_params.py | 58 -
.../HDFS/3.0.0.3.0/package/scripts/utils.py | 428 --
.../HDFS/3.0.0.3.0/package/scripts/zkfc_slave.py | 184 -
.../package/templates/exclude_hosts_list.j2 | 21 -
.../HDFS/3.0.0.3.0/package/templates/hdfs.conf.j2 | 35 -
.../package/templates/hdfs_dn_jaas.conf.j2 | 27 -
.../3.0.0.3.0/package/templates/hdfs_jaas.conf.j2 | 27 -
.../package/templates/hdfs_jn_jaas.conf.j2 | 27 -
.../package/templates/hdfs_nn_jaas.conf.j2 | 27 -
.../package/templates/include_hosts_list.j2 | 21 -
.../HDFS/3.0.0.3.0/package/templates/slaves.j2 | 21 -
.../HDFS/3.0.0.3.0/quicklinks/quicklinks.json | 80 -
.../HDFS/3.0.0.3.0/service_advisor.py | 676 --
.../HDFS/3.0.0.3.0/themes/theme.json | 179 -
.../common-services/HDFS/3.0.0.3.0/widgets.json | 649 --
.../package/scripts/hive_server_interactive.py | 3 +-
.../common-services/HIVE/2.1.0.3.0/alerts.json | 232 -
.../2.1.0.3.0/configuration/beeline-log4j2.xml | 80 -
.../HIVE/2.1.0.3.0/configuration/hcat-env.xml | 60 -
.../hive-atlas-application.properties.xml | 61 -
.../HIVE/2.1.0.3.0/configuration/hive-env.xml | 509 --
.../2.1.0.3.0/configuration/hive-exec-log4j.xml | 114 -
.../2.1.0.3.0/configuration/hive-exec-log4j2.xml | 101 -
.../configuration/hive-interactive-env.xml | 331 -
.../configuration/hive-interactive-site.xml | 762 --
.../HIVE/2.1.0.3.0/configuration/hive-log4j.xml | 147 -
.../HIVE/2.1.0.3.0/configuration/hive-log4j2.xml | 131 -
.../configuration/hive-logsearch-conf.xml | 117 -
.../HIVE/2.1.0.3.0/configuration/hive-site.xml | 2052 -----
.../2.1.0.3.0/configuration/hivemetastore-site.xml | 46 -
.../configuration/hiveserver2-interactive-site.xml | 55 -
.../2.1.0.3.0/configuration/hiveserver2-site.xml | 110 -
.../2.1.0.3.0/configuration/llap-cli-log4j2.xml | 148 -
.../2.1.0.3.0/configuration/llap-daemon-log4j.xml | 210 -
.../2.1.0.3.0/configuration/parquet-logging.xml | 106 -
.../2.1.0.3.0/configuration/ranger-hive-audit.xml | 130 -
.../ranger-hive-plugin-properties.xml | 127 -
.../configuration/ranger-hive-policymgr-ssl.xml | 66 -
.../configuration/ranger-hive-security.xml | 83 -
.../configuration/tez-interactive-site.xml | 257 -
.../HIVE/2.1.0.3.0/configuration/webhcat-env.xml | 57 -
.../HIVE/2.1.0.3.0/configuration/webhcat-log4j.xml | 104 -
.../HIVE/2.1.0.3.0/configuration/webhcat-site.xml | 193 -
.../common-services/HIVE/2.1.0.3.0/kerberos.json | 161 -
.../common-services/HIVE/2.1.0.3.0/metainfo.xml | 536 --
.../alerts/alert_hive_interactive_thrift_port.py | 230 -
.../package/alerts/alert_hive_metastore.py | 273 -
.../package/alerts/alert_hive_thrift_port.py | 285 -
.../package/alerts/alert_llap_app_status.py | 299 -
.../package/alerts/alert_webhcat_server.py | 228 -
.../package/etc/hive-schema-0.12.0.mysql.sql | 777 --
.../package/etc/hive-schema-0.12.0.oracle.sql | 718 --
.../package/etc/hive-schema-0.12.0.postgres.sql | 1406 ----
.../package/etc/hive-schema-0.13.0.mysql.sql | 889 ---
.../package/etc/hive-schema-0.13.0.oracle.sql | 835 ---
.../package/etc/hive-schema-0.13.0.postgres.sql | 1538 ----
.../etc/upgrade-0.12.0-to-0.13.0.oracle.sql | 165 -
.../package/etc/upgrade-0.13.0.oracle.sql | 38 -
.../HIVE/2.1.0.3.0/package/files/addMysqlUser.sh | 39 -
.../HIVE/2.1.0.3.0/package/files/hcatSmoke.sh | 41 -
.../HIVE/2.1.0.3.0/package/files/hiveSmoke.sh | 24 -
.../HIVE/2.1.0.3.0/package/files/hiveTezSetup.cmd | 58 -
.../HIVE/2.1.0.3.0/package/files/hiveserver2.sql | 23 -
.../2.1.0.3.0/package/files/hiveserver2Smoke.sh | 32 -
.../2.1.0.3.0/package/files/removeMysqlUser.sh | 33 -
.../HIVE/2.1.0.3.0/package/files/startMetastore.sh | 25 -
.../HIVE/2.1.0.3.0/package/files/templetonSmoke.sh | 101 -
.../HIVE/2.1.0.3.0/package/scripts/__init__.py | 19 -
.../HIVE/2.1.0.3.0/package/scripts/hcat.py | 87 -
.../HIVE/2.1.0.3.0/package/scripts/hcat_client.py | 78 -
.../package/scripts/hcat_service_check.py | 89 -
.../HIVE/2.1.0.3.0/package/scripts/hive.py | 561 --
.../HIVE/2.1.0.3.0/package/scripts/hive_client.py | 63 -
.../2.1.0.3.0/package/scripts/hive_interactive.py | 359 -
.../2.1.0.3.0/package/scripts/hive_metastore.py | 211 -
.../HIVE/2.1.0.3.0/package/scripts/hive_server.py | 164 -
.../package/scripts/hive_server_interactive.py | 610 --
.../package/scripts/hive_server_upgrade.py | 140 -
.../HIVE/2.1.0.3.0/package/scripts/hive_service.py | 181 -
.../package/scripts/hive_service_interactive.py | 109 -
.../HIVE/2.1.0.3.0/package/scripts/mysql_server.py | 63 -
.../2.1.0.3.0/package/scripts/mysql_service.py | 51 -
.../HIVE/2.1.0.3.0/package/scripts/mysql_users.py | 72 -
.../HIVE/2.1.0.3.0/package/scripts/mysql_utils.py | 35 -
.../HIVE/2.1.0.3.0/package/scripts/params.py | 30 -
.../HIVE/2.1.0.3.0/package/scripts/params_linux.py | 853 ---
.../2.1.0.3.0/package/scripts/params_windows.py | 75 -
.../2.1.0.3.0/package/scripts/service_check.py | 192 -
.../2.1.0.3.0/package/scripts/setup_ranger_hive.py | 156 -
.../scripts/setup_ranger_hive_interactive.py | 77 -
.../2.1.0.3.0/package/scripts/status_params.py | 118 -
.../HIVE/2.1.0.3.0/package/scripts/webhcat.py | 139 -
.../2.1.0.3.0/package/scripts/webhcat_server.py | 95 -
.../2.1.0.3.0/package/scripts/webhcat_service.py | 100 -
.../package/scripts/webhcat_service_check.py | 133 -
.../hadoop-metrics2-hivemetastore.properties.j2 | 58 -
.../hadoop-metrics2-hiveserver2.properties.j2 | 57 -
.../templates/hadoop-metrics2-llapdaemon.j2 | 56 -
.../templates/hadoop-metrics2-llaptaskscheduler.j2 | 56 -
.../HIVE/2.1.0.3.0/package/templates/hive.conf.j2 | 35 -
.../package/templates/startHiveserver2.sh.j2 | 24 -
.../templates/startHiveserver2Interactive.sh.j2 | 24 -
.../package/templates/templeton_smoke.pig.j2 | 24 -
.../package/templates/zkmigrator_jaas.conf.j2 | 26 -
.../HIVE/2.1.0.3.0/quicklinks/quicklinks.json | 62 -
.../HIVE/2.1.0.3.0/service_advisor.py | 1166 ---
.../HIVE/2.1.0.3.0/themes/theme.json | 753 --
.../common-services/KAFKA/0.10.0.3.0/alerts.json | 32 -
.../0.10.0.3.0/configuration/kafka-broker.xml | 559 --
.../KAFKA/0.10.0.3.0/configuration/kafka-env.xml | 117 -
.../KAFKA/0.10.0.3.0/configuration/kafka-log4j.xml | 170 -
.../configuration/kafka_client_jaas_conf.xml | 41 -
.../0.10.0.3.0/configuration/kafka_jaas_conf.xml | 59 -
.../configuration/ranger-kafka-audit.xml | 130 -
.../ranger-kafka-plugin-properties.xml | 154 -
.../configuration/ranger-kafka-policymgr-ssl.xml | 66 -
.../configuration/ranger-kafka-security.xml | 64 -
.../common-services/KAFKA/0.10.0.3.0/kerberos.json | 82 -
.../common-services/KAFKA/0.10.0.3.0/metainfo.xml | 114 -
.../common-services/KAFKA/0.10.0.3.0/metrics.json | 239 -
.../KAFKA/0.10.0.3.0/package/scripts/kafka.py | 257 -
.../0.10.0.3.0/package/scripts/kafka_broker.py | 145 -
.../KAFKA/0.10.0.3.0/package/scripts/params.py | 341 -
.../0.10.0.3.0/package/scripts/service_check.py | 70 -
.../package/scripts/setup_ranger_kafka.py | 98 -
.../0.10.0.3.0/package/scripts/status_params.py | 26 -
.../KAFKA/0.10.0.3.0/package/scripts/upgrade.py | 75 -
.../KAFKA/0.10.0.3.0/package/scripts/utils.py | 38 -
.../package/templates/input.config-kafka.json.j2 | 92 -
.../0.10.0.3.0/package/templates/kafka.conf.j2 | 35 -
.../package/templates/kafka_client_jaas.conf.j2 | 29 -
.../package/templates/kafka_jaas.conf.j2 | 41 -
.../package/templates/tools-log4j.properties.j2 | 21 -
.../KAFKA/0.10.0.3.0/role_command_order.json | 7 -
.../KAFKA/0.10.0.3.0/service_advisor.py | 368 -
.../common-services/KAFKA/0.10.0.3.0/widgets.json | 182 -
.../KAFKA/0.8.1/package/scripts/kafka.py | 6 +-
.../1.10.3-10/configuration/kerberos-env.xml | 2 +-
.../1.10.3-30/configuration/kerberos-env.xml | 2 +-
.../common-services/KNOX/0.5.0.3.0/alerts.json | 32 -
.../0.5.0.3.0/configuration/admin-topology.xml | 97 -
.../KNOX/0.5.0.3.0/configuration/gateway-log4j.xml | 110 -
.../KNOX/0.5.0.3.0/configuration/gateway-site.xml | 71 -
.../KNOX/0.5.0.3.0/configuration/knox-env.xml | 93 -
.../0.5.0.3.0/configuration/knoxsso-topology.xml | 126 -
.../KNOX/0.5.0.3.0/configuration/ldap-log4j.xml | 93 -
.../0.5.0.3.0/configuration/ranger-knox-audit.xml | 132 -
.../ranger-knox-plugin-properties.xml | 132 -
.../configuration/ranger-knox-policymgr-ssl.xml | 66 -
.../configuration/ranger-knox-security.xml | 64 -
.../KNOX/0.5.0.3.0/configuration/topology.xml | 174 -
.../KNOX/0.5.0.3.0/configuration/users-ldif.xml | 140 -
.../common-services/KNOX/0.5.0.3.0/kerberos.json | 82 -
.../common-services/KNOX/0.5.0.3.0/metainfo.xml | 109 -
.../0.5.0.3.0/package/files/validateKnoxStatus.py | 43 -
.../KNOX/0.5.0.3.0/package/scripts/knox.py | 192 -
.../KNOX/0.5.0.3.0/package/scripts/knox_gateway.py | 215 -
.../KNOX/0.5.0.3.0/package/scripts/knox_ldap.py | 59 -
.../KNOX/0.5.0.3.0/package/scripts/params.py | 29 -
.../KNOX/0.5.0.3.0/package/scripts/params_linux.py | 566 --
.../0.5.0.3.0/package/scripts/params_windows.py | 71 -
.../0.5.0.3.0/package/scripts/service_check.py | 96 -
.../0.5.0.3.0/package/scripts/setup_ranger_knox.py | 129 -
.../0.5.0.3.0/package/scripts/status_params.py | 59 -
.../KNOX/0.5.0.3.0/package/scripts/upgrade.py | 118 -
.../package/templates/input.config-knox.json.j2 | 60 -
.../package/templates/krb5JAASLogin.conf.j2 | 27 -
.../KNOX/0.5.0.3.0/role_command_order.json | 7 -
.../KNOX/0.5.0.3.0/service_advisor.py | 253 -
.../0.5.0/configuration/logsearch-common-env.xml | 20 +
.../LOGSEARCH/0.5.0/package/scripts/params.py | 7 +
.../0.5.0/package/scripts/setup_logsearch.py | 3 +-
.../LOGSEARCH/0.5.0/properties/logfeeder-env.sh.j2 | 4 +
.../LOGSEARCH/0.5.0/properties/logsearch-env.sh.j2 | 4 +
.../common-services/OOZIE/4.2.0.3.0/alerts.json | 45 -
.../OOZIE/4.2.0.3.0/configuration/oozie-env.xml | 266 -
.../OOZIE/4.2.0.3.0/configuration/oozie-log4j.xml | 149 -
.../OOZIE/4.2.0.3.0/configuration/oozie-site.xml | 254 -
.../common-services/OOZIE/4.2.0.3.0/kerberos.json | 74 -
.../common-services/OOZIE/4.2.0.3.0/metainfo.xml | 196 -
.../package/alerts/alert_check_oozie_server.py | 244 -
.../OOZIE/4.2.0.3.0/package/files/oozieSmoke2.sh | 84 -
.../package/files/prepareOozieHdfsDirectories.sh | 42 -
.../OOZIE/4.2.0.3.0/package/files/wrap_ooziedb.sh | 31 -
.../package/scripts/check_oozie_server_status.py | 38 -
.../OOZIE/4.2.0.3.0/package/scripts/oozie.py | 509 --
.../4.2.0.3.0/package/scripts/oozie_client.py | 74 -
.../4.2.0.3.0/package/scripts/oozie_server.py | 155 -
.../package/scripts/oozie_server_upgrade.py | 261 -
.../4.2.0.3.0/package/scripts/oozie_service.py | 188 -
.../OOZIE/4.2.0.3.0/package/scripts/params.py | 39 -
.../4.2.0.3.0/package/scripts/params_linux.py | 370 -
.../4.2.0.3.0/package/scripts/params_windows.py | 34 -
.../4.2.0.3.0/package/scripts/service_check.py | 140 -
.../4.2.0.3.0/package/scripts/status_params.py | 65 -
.../4.2.0.3.0/package/templates/adminusers.txt.j2 | 28 -
.../package/templates/input.config-oozie.json.j2 | 48 -
.../package/templates/oozie-log4j.properties.j2 | 93 -
.../4.2.0.3.0/package/templates/oozie.conf.j2 | 35 -
.../package/templates/zkmigrator_jaas.conf.j2 | 26 -
.../OOZIE/4.2.0.3.0/quicklinks/quicklinks.json | 46 -
.../OOZIE/4.2.0.3.0/role_command_order.json | 9 -
.../OOZIE/4.2.0.3.0/service_advisor.py | 314 -
.../OOZIE/4.2.0.3.0/themes/theme.json | 116 -
.../PIG/0.16.1.3.0/configuration/pig-env.xml | 41 -
.../PIG/0.16.1.3.0/configuration/pig-log4j.xml | 65 -
.../0.16.1.3.0/configuration/pig-properties.xml | 93 -
.../common-services/PIG/0.16.1.3.0/kerberos.json | 18 -
.../common-services/PIG/0.16.1.3.0/metainfo.xml | 107 -
.../PIG/0.16.1.3.0/package/files/pigSmoke.sh | 18 -
.../PIG/0.16.1.3.0/package/scripts/params.py | 31 -
.../PIG/0.16.1.3.0/package/scripts/params_linux.py | 104 -
.../0.16.1.3.0/package/scripts/params_windows.py | 45 -
.../PIG/0.16.1.3.0/package/scripts/pig.py | 86 -
.../PIG/0.16.1.3.0/package/scripts/pig_client.py | 66 -
.../0.16.1.3.0/package/scripts/service_check.py | 142 -
.../PIG/0.16.1.3.0/service_advisor.py | 171 -
.../common-services/RANGER/1.0.0.3.0/alerts.json | 76 -
.../RANGER/1.0.0.3.0/configuration/admin-log4j.xml | 131 -
.../1.0.0.3.0/configuration/admin-properties.xml | 161 -
.../1.0.0.3.0/configuration/atlas-tagsync-ssl.xml | 72 -
.../1.0.0.3.0/configuration/ranger-admin-site.xml | 751 --
.../RANGER/1.0.0.3.0/configuration/ranger-env.xml | 536 --
.../configuration/ranger-solr-configuration.xml | 59 -
.../configuration/ranger-tagsync-policymgr-ssl.xml | 72 -
.../configuration/ranger-tagsync-site.xml | 201 -
.../1.0.0.3.0/configuration/ranger-ugsync-site.xml | 578 --
.../tagsync-application-properties.xml | 62 -
.../1.0.0.3.0/configuration/tagsync-log4j.xml | 89 -
.../1.0.0.3.0/configuration/usersync-log4j.xml | 88 -
.../common-services/RANGER/1.0.0.3.0/kerberos.json | 159 -
.../common-services/RANGER/1.0.0.3.0/metainfo.xml | 177 -
.../alerts/alert_ranger_admin_passwd_check.py | 195 -
.../RANGER/1.0.0.3.0/package/scripts/params.py | 450 --
.../1.0.0.3.0/package/scripts/ranger_admin.py | 217 -
.../1.0.0.3.0/package/scripts/ranger_service.py | 69 -
.../1.0.0.3.0/package/scripts/ranger_tagsync.py | 147 -
.../1.0.0.3.0/package/scripts/ranger_usersync.py | 117 -
.../1.0.0.3.0/package/scripts/service_check.py | 49 -
.../1.0.0.3.0/package/scripts/setup_ranger_xml.py | 853 ---
.../1.0.0.3.0/package/scripts/status_params.py | 39 -
.../RANGER/1.0.0.3.0/package/scripts/upgrade.py | 29 -
.../package/templates/input.config-ranger.json.j2 | 79 -
.../package/templates/ranger_admin_pam.j2 | 22 -
.../package/templates/ranger_remote_pam.j2 | 22 -
.../package/templates/ranger_solr_jaas_conf.j2 | 26 -
.../1.0.0.3.0/properties/ranger-solrconfig.xml.j2 | 1874 -----
.../RANGER/1.0.0.3.0/quicklinks/quicklinks.json | 41 -
.../RANGER/1.0.0.3.0/role_command_order.json | 9 -
.../RANGER/1.0.0.3.0/service_advisor.py | 774 --
.../RANGER/1.0.0.3.0/themes/theme_version_1.json | 1861 -----
.../RANGER_KMS/1.0.0.3.0/alerts.json | 32 -
.../1.0.0.3.0/configuration/dbks-site.xml | 206 -
.../RANGER_KMS/1.0.0.3.0/configuration/kms-env.xml | 125 -
.../1.0.0.3.0/configuration/kms-log4j.xml | 118 -
.../1.0.0.3.0/configuration/kms-properties.xml | 166 -
.../1.0.0.3.0/configuration/kms-site.xml | 107 -
.../1.0.0.3.0/configuration/ranger-kms-audit.xml | 118 -
.../configuration/ranger-kms-policymgr-ssl.xml | 68 -
.../configuration/ranger-kms-security.xml | 64 -
.../1.0.0.3.0/configuration/ranger-kms-site.xml | 110 -
.../RANGER_KMS/1.0.0.3.0/kerberos.json | 81 -
.../RANGER_KMS/1.0.0.3.0/metainfo.xml | 117 -
.../RANGER_KMS/1.0.0.3.0/package/scripts/kms.py | 675 --
.../1.0.0.3.0/package/scripts/kms_server.py | 115 -
.../1.0.0.3.0/package/scripts/kms_service.py | 58 -
.../RANGER_KMS/1.0.0.3.0/package/scripts/params.py | 334 -
.../1.0.0.3.0/package/scripts/service_check.py | 41 -
.../1.0.0.3.0/package/scripts/status_params.py | 36 -
.../1.0.0.3.0/package/scripts/upgrade.py | 29 -
.../templates/input.config-ranger-kms.json.j2 | 48 -
.../RANGER_KMS/1.0.0.3.0/role_command_order.json | 7 -
.../RANGER_KMS/1.0.0.3.0/service_advisor.py | 359 -
.../1.0.0.3.0/themes/theme_version_1.json | 409 -
.../0.91.0.3.0/configuration/slider-client.xml | 56 -
.../SLIDER/0.91.0.3.0/configuration/slider-env.xml | 46 -
.../0.91.0.3.0/configuration/slider-log4j.xml | 92 -
.../SLIDER/0.91.0.3.0/kerberos.json | 18 -
.../common-services/SLIDER/0.91.0.3.0/metainfo.xml | 118 -
.../0.91.0.3.0/package/files/hbaseSmokeVerify.sh | 34 -
.../SLIDER/0.91.0.3.0/package/scripts/__init__.py | 19 -
.../SLIDER/0.91.0.3.0/package/scripts/params.py | 54 -
.../0.91.0.3.0/package/scripts/params_linux.py | 80 -
.../0.91.0.3.0/package/scripts/params_windows.py | 45 -
.../0.91.0.3.0/package/scripts/service_check.py | 63 -
.../SLIDER/0.91.0.3.0/package/scripts/slider.py | 100 -
.../0.91.0.3.0/package/scripts/slider_client.py | 61 -
.../package/templates/storm-slider-env.sh.j2 | 38 -
.../SLIDER/0.91.0.3.0/service_advisor.py | 134 -
.../common-services/SPARK/2.2.0/alerts.json | 32 -
.../SPARK/2.2.0/configuration/livy-conf.xml | 114 -
.../SPARK/2.2.0/configuration/livy-env.xml | 107 -
.../2.2.0/configuration/livy-log4j-properties.xml | 42 -
.../2.2.0/configuration/livy-spark-blacklist.xml | 52 -
.../SPARK/2.2.0/configuration/spark-defaults.xml | 115 -
.../SPARK/2.2.0/configuration/spark-env.xml | 157 -
.../configuration/spark-hive-site-override.xml | 59 -
.../2.2.0/configuration/spark-log4j-properties.xml | 46 -
.../2.2.0/configuration/spark-logsearch-conf.xml | 98 -
.../configuration/spark-metrics-properties.xml | 165 -
.../configuration/spark-thrift-fairscheduler.xml | 36 -
.../2.2.0/configuration/spark-thrift-sparkconf.xml | 160 -
.../common-services/SPARK/2.2.0/kerberos.json | 127 -
.../common-services/SPARK/2.2.0/metainfo.xml | 292 -
.../2.2.0/package/scripts/job_history_server.py | 103 -
.../SPARK/2.2.0/package/scripts/livy_server.py | 146 -
.../SPARK/2.2.0/package/scripts/livy_service.py | 52 -
.../SPARK/2.2.0/package/scripts/params.py | 270 -
.../SPARK/2.2.0/package/scripts/service_check.py | 62 -
.../SPARK/2.2.0/package/scripts/setup_livy.py | 88 -
.../SPARK/2.2.0/package/scripts/setup_spark.py | 117 -
.../SPARK/2.2.0/package/scripts/spark_client.py | 57 -
.../SPARK/2.2.0/package/scripts/spark_service.py | 146 -
.../2.2.0/package/scripts/spark_thrift_server.py | 86 -
.../SPARK/2.2.0/package/scripts/status_params.py | 45 -
.../SPARK/2.2.0/quicklinks/quicklinks.json | 27 -
.../common-services/SPARK/2.2.0/service_advisor.py | 243 -
.../sqoop-atlas-application.properties.xml | 47 -
.../SQOOP/1.4.4.3.0/configuration/sqoop-env.xml | 93 -
.../SQOOP/1.4.4.3.0/configuration/sqoop-site.xml | 38 -
.../common-services/SQOOP/1.4.4.3.0/kerberos.json | 20 -
.../common-services/SQOOP/1.4.4.3.0/metainfo.xml | 108 -
.../SQOOP/1.4.4.3.0/package/scripts/__init__.py | 19 -
.../SQOOP/1.4.4.3.0/package/scripts/params.py | 27 -
.../1.4.4.3.0/package/scripts/params_linux.py | 137 -
.../1.4.4.3.0/package/scripts/params_windows.py | 30 -
.../1.4.4.3.0/package/scripts/service_check.py | 59 -
.../SQOOP/1.4.4.3.0/package/scripts/sqoop.py | 124 -
.../1.4.4.3.0/package/scripts/sqoop_client.py | 61 -
.../SQOOP/1.4.4.3.0/role_command_order.json | 6 -
.../SQOOP/1.4.4.3.0/service_advisor.py | 197 -
.../common-services/STORM/1.0.1.3.0/alerts.json | 145 -
.../1.0.1.3.0/configuration/ranger-storm-audit.xml | 133 -
.../ranger-storm-plugin-properties.xml | 121 -
.../configuration/ranger-storm-policymgr-ssl.xml | 70 -
.../configuration/ranger-storm-security.xml | 67 -
.../storm-atlas-application.properties.xml | 31 -
.../configuration/storm-cluster-log4j.xml | 133 -
.../STORM/1.0.1.3.0/configuration/storm-env.xml | 171 -
.../STORM/1.0.1.3.0/configuration/storm-site.xml | 1002 ---
.../1.0.1.3.0/configuration/storm-worker-log4j.xml | 189 -
.../common-services/STORM/1.0.1.3.0/kerberos.json | 139 -
.../common-services/STORM/1.0.1.3.0/metainfo.xml | 180 -
.../common-services/STORM/1.0.1.3.0/metrics.json | 1202 ---
.../package/alerts/check_supervisor_process_win.py | 50 -
.../STORM/1.0.1.3.0/package/files/wordCount.jar | Bin 690588 -> 0 bytes
.../STORM/1.0.1.3.0/package/scripts/drpc_server.py | 86 -
.../STORM/1.0.1.3.0/package/scripts/nimbus.py | 110 -
.../STORM/1.0.1.3.0/package/scripts/nimbus_prod.py | 75 -
.../STORM/1.0.1.3.0/package/scripts/pacemaker.py | 85 -
.../STORM/1.0.1.3.0/package/scripts/params.py | 28 -
.../1.0.1.3.0/package/scripts/params_linux.py | 422 --
.../1.0.1.3.0/package/scripts/params_windows.py | 60 -
.../STORM/1.0.1.3.0/package/scripts/rest_api.py | 82 -
.../STORM/1.0.1.3.0/package/scripts/service.py | 95 -
.../1.0.1.3.0/package/scripts/service_check.py | 79 -
.../package/scripts/setup_ranger_storm.py | 142 -
.../1.0.1.3.0/package/scripts/status_params.py | 83 -
.../STORM/1.0.1.3.0/package/scripts/storm.py | 182 -
.../1.0.1.3.0/package/scripts/storm_upgrade.py | 177 -
.../1.0.1.3.0/package/scripts/storm_yaml_utils.py | 53 -
.../STORM/1.0.1.3.0/package/scripts/supervisor.py | 111 -
.../1.0.1.3.0/package/scripts/supervisor_prod.py | 78 -
.../package/scripts/supervisord_service.py | 34 -
.../STORM/1.0.1.3.0/package/scripts/ui_server.py | 132 -
.../package/templates/client_jaas.conf.j2 | 33 -
.../1.0.1.3.0/package/templates/config.yaml.j2 | 72 -
.../package/templates/input.config-storm.json.j2 | 78 -
.../package/templates/storm-metrics2.properties.j2 | 32 -
.../1.0.1.3.0/package/templates/storm.conf.j2 | 35 -
.../1.0.1.3.0/package/templates/storm_jaas.conf.j2 | 65 -
.../package/templates/worker-launcher.cfg.j2 | 19 -
.../STORM/1.0.1.3.0/quicklinks/quicklinks.json | 45 -
.../STORM/1.0.1.3.0/role_command_order.json | 13 -
.../STORM/1.0.1.3.0/service_advisor.py | 388 -
.../common-services/STORM/1.0.1.3.0/widgets.json | 127 -
.../TEZ/0.9.0.3.0/configuration/tez-env.xml | 97 -
.../TEZ/0.9.0.3.0/configuration/tez-site.xml | 523 --
.../common-services/TEZ/0.9.0.3.0/kerberos.json | 25 -
.../common-services/TEZ/0.9.0.3.0/metainfo.xml | 114 -
.../TEZ/0.9.0.3.0/package/scripts/params.py | 29 -
.../TEZ/0.9.0.3.0/package/scripts/params_linux.py | 116 -
.../0.9.0.3.0/package/scripts/params_windows.py | 54 -
.../TEZ/0.9.0.3.0/package/scripts/pre_upgrade.py | 59 -
.../TEZ/0.9.0.3.0/package/scripts/service_check.py | 111 -
.../TEZ/0.9.0.3.0/package/scripts/tez.py | 80 -
.../TEZ/0.9.0.3.0/package/scripts/tez_client.py | 130 -
.../TEZ/0.9.0.3.0/service_advisor.py | 371 -
.../YARN/2.1.0.2.0/configuration/yarn-env.xml | 5 +
.../YARN/2.1.0.2.0/package/scripts/params_linux.py | 4 +-
.../YARN/3.0.0.3.0/MAPREDUCE2_metrics.json | 2596 -------
.../YARN/3.0.0.3.0/YARN_metrics.json | 3486 ---------
.../YARN/3.0.0.3.0/YARN_widgets.json | 670 --
.../common-services/YARN/3.0.0.3.0/alerts.json | 392 -
.../3.0.0.3.0/configuration-mapred/mapred-env.xml | 112 -
.../configuration-mapred/mapred-logsearch-conf.xml | 80 -
.../3.0.0.3.0/configuration-mapred/mapred-site.xml | 540 --
.../3.0.0.3.0/configuration/capacity-scheduler.xml | 183 -
.../3.0.0.3.0/configuration/ranger-yarn-audit.xml | 186 -
.../ranger-yarn-plugin-properties.xml | 92 -
.../configuration/ranger-yarn-policymgr-ssl.xml | 66 -
.../configuration/ranger-yarn-security.xml | 58 -
.../YARN/3.0.0.3.0/configuration/yarn-env.xml | 323 -
.../YARN/3.0.0.3.0/configuration/yarn-log4j.xml | 103 -
.../configuration/yarn-logsearch-conf.xml | 104 -
.../YARN/3.0.0.3.0/configuration/yarn-site.xml | 1197 ---
.../common-services/YARN/3.0.0.3.0/kerberos.json | 294 -
.../common-services/YARN/3.0.0.3.0/metainfo.xml | 407 -
.../package/alerts/alert_nodemanager_health.py | 209 -
.../package/alerts/alert_nodemanagers_summary.py | 219 -
.../files/validateYarnComponentStatusWindows.py | 161 -
.../YARN/3.0.0.3.0/package/scripts/__init__.py | 20 -
.../package/scripts/application_timeline_server.py | 97 -
.../3.0.0.3.0/package/scripts/historyserver.py | 132 -
.../YARN/3.0.0.3.0/package/scripts/install_jars.py | 99 -
.../package/scripts/mapred_service_check.py | 172 -
.../3.0.0.3.0/package/scripts/mapreduce2_client.py | 93 -
.../YARN/3.0.0.3.0/package/scripts/nodemanager.py | 102 -
.../package/scripts/nodemanager_upgrade.py | 74 -
.../YARN/3.0.0.3.0/package/scripts/params.py | 32 -
.../YARN/3.0.0.3.0/package/scripts/params_linux.py | 588 --
.../3.0.0.3.0/package/scripts/params_windows.py | 70 -
.../3.0.0.3.0/package/scripts/resourcemanager.py | 257 -
.../YARN/3.0.0.3.0/package/scripts/service.py | 113 -
.../3.0.0.3.0/package/scripts/service_check.py | 185 -
.../3.0.0.3.0/package/scripts/setup_ranger_yarn.py | 71 -
.../3.0.0.3.0/package/scripts/status_params.py | 49 -
.../YARN/3.0.0.3.0/package/scripts/yarn.py | 542 --
.../YARN/3.0.0.3.0/package/scripts/yarn_client.py | 63 -
.../package/templates/container-executor.cfg.j2 | 40 -
.../package/templates/exclude_hosts_list.j2 | 21 -
.../package/templates/include_hosts_list.j2 | 21 -
.../package/templates/mapred_jaas.conf.j2 | 28 -
.../3.0.0.3.0/package/templates/mapreduce.conf.j2 | 35 -
.../package/templates/taskcontroller.cfg.j2 | 38 -
.../YARN/3.0.0.3.0/package/templates/yarn.conf.j2 | 35 -
.../package/templates/yarn_ats_jaas.conf.j2 | 27 -
.../3.0.0.3.0/package/templates/yarn_jaas.conf.j2 | 36 -
.../package/templates/yarn_nm_jaas.conf.j2 | 27 -
.../3.0.0.3.0/quicklinks-mapred/quicklinks.json | 80 -
.../YARN/3.0.0.3.0/quicklinks/quicklinks.json | 80 -
.../YARN/3.0.0.3.0/service_advisor.py | 2009 -----
.../YARN/3.0.0.3.0/themes-mapred/theme.json | 132 -
.../YARN/3.0.0.3.0/themes/theme.json | 250 -
.../custom_actions/scripts/install_packages.py | 8 +-
.../src/main/resources/scripts/Ambaripreupload.py | 2 +-
.../resources/scripts/check_ambari_permissions.py | 8 +-
.../src/main/resources/scripts/kerberos-setup.sh | 4 +-
.../before-INSTALL/scripts/repo_initialization.py | 9 +-
.../stacks/HDP/2.0.6/services/stack_advisor.py | 49 -
.../2.1/services/YARN/configuration/yarn-env.xml | 1 +
.../2.2/services/HDFS/configuration/hadoop-env.xml | 2 +
.../2.3/services/HDFS/configuration/hadoop-env.xml | 2 +
.../2.3/services/YARN/configuration/yarn-env.xml | 1 +
.../2.4/services/HDFS/configuration/hadoop-env.xml | 2 +
.../2.6/services/HDFS/configuration/hadoop-env.xml | 2 +
.../2.6/services/YARN/configuration/yarn-env.xml | 1 +
.../stacks/HDP/3.0/configuration/cluster-env.xml | 359 -
.../src/main/resources/stacks/HDP/3.0/metainfo.xml | 24 -
.../stacks/HDP/3.0/properties/stack_features.json | 379 -
.../stacks/HDP/3.0/properties/stack_packages.json | 1162 ---
.../stacks/HDP/3.0/properties/stack_tools.json | 14 -
.../resources/stacks/HDP/3.0/repos/repoinfo.xml | 132 -
.../stacks/HDP/3.0/role_command_order.json | 172 -
.../stacks/HDP/3.0/services/ATLAS/metainfo.xml | 27 -
.../stacks/HDP/3.0/services/HBASE/metainfo.xml | 26 -
.../3.0/services/HDFS/configuration/hadoop-env.xml | 177 -
.../stacks/HDP/3.0/services/HDFS/metainfo.xml | 115 -
.../stacks/HDP/3.0/services/HIVE/metainfo.xml | 27 -
.../stacks/HDP/3.0/services/KAFKA/metainfo.xml | 27 -
.../stacks/HDP/3.0/services/KERBEROS/metainfo.xml | 26 -
.../stacks/HDP/3.0/services/KNOX/metainfo.xml | 27 -
.../stacks/HDP/3.0/services/OOZIE/metainfo.xml | 27 -
.../stacks/HDP/3.0/services/PIG/metainfo.xml | 27 -
.../stacks/HDP/3.0/services/RANGER/metainfo.xml | 27 -
.../HDP/3.0/services/RANGER_KMS/metainfo.xml | 27 -
.../stacks/HDP/3.0/services/SLIDER/metainfo.xml | 51 -
.../stacks/HDP/3.0/services/SPARK/metainfo.xml | 32 -
.../stacks/HDP/3.0/services/SQOOP/metainfo.xml | 27 -
.../stacks/HDP/3.0/services/STORM/metainfo.xml | 27 -
.../stacks/HDP/3.0/services/TEZ/metainfo.xml | 27 -
.../YARN/configuration-mapred/mapred-env.xml | 51 -
.../YARN/configuration-mapred/mapred-site.xml | 78 -
.../3.0/services/YARN/configuration/yarn-site.xml | 48 -
.../stacks/HDP/3.0/services/YARN/metainfo.xml | 84 -
.../stacks/HDP/3.0/services/ZEPPELIN/metainfo.xml | 27 -
.../stacks/HDP/3.0/services/ZOOKEEPER/metainfo.xml | 52 -
.../src/main/resources/stacks/stack_advisor.py | 48 +
.../actionmanager/ExecutionCommandWrapperTest.java | 2 +-
.../server/checks/InstallPackagesCheckTest.java | 8 +-
.../server/configuration/ConfigurationTest.java | 24 +
.../AmbariCustomCommandExecutionHelperTest.java | 19 +-
.../controller/AmbariManagementControllerTest.java | 2 +-
.../ClusterStackVersionResourceProviderTest.java | 121 +-
...tibleRepositoryVersionResourceProviderTest.java | 14 +-
.../HostStackVersionResourceProviderTest.java | 40 +-
.../RepositoryVersionResourceProviderTest.java | 78 +-
...ComponentConfigurationResourceProviderTest.java | 2 +-
.../internal/UpgradeResourceProviderTest.java | 51 +-
.../UpgradeSummaryResourceProviderTest.java | 2 +-
.../internal/UserResourceProviderTest.java | 2 +-
.../logging/LogSearchDataRetrievalServiceTest.java | 21 +-
.../LoggingRequestHelperFactoryImplTest.java | 46 +
.../logging/LoggingRequestHelperImplTest.java | 49 +-
.../apache/ambari/server/orm/OrmTestHelper.java | 21 +-
.../apache/ambari/server/orm/dao/CrudDAOTest.java | 3 +-
.../ambari/server/orm/dao/HostVersionDAOTest.java | 1 +
.../server/orm/dao/RepositoryVersionDAOTest.java | 15 +-
.../pam/AmbariPamAuthenticationProviderTest.java | 15 +-
...apAuthenticationProviderForDNWithSpaceTest.java | 37 +-
...AuthenticationProviderForDuplicateUserTest.java | 20 +-
.../AmbariLdapAuthenticationProviderTest.java | 46 +-
.../security/ldap/AmbariLdapDataPopulatorTest.java | 297 +-
.../upgrades/ComponentVersionCheckActionTest.java | 17 +-
.../apache/ambari/server/stack/HostsTypeTest.java | 82 +
.../ambari/server/stack/NameServiceTest.java | 136 +
.../apache/ambari/server/stack/RepoUtilTest.java | 57 +-
.../UpdateActiveRepoVersionOnStartupTest.java | 37 +-
.../ambari/server/state/ServiceComponentTest.java | 5 +-
.../ambari/server/state/UpgradeHelperTest.java | 125 +-
.../services/RetryUpgradeActionServiceTest.java | 4 +-
.../stack/upgrade/RepositoryVersionHelperTest.java | 58 -
.../server/topology/AsyncCallableServiceTest.java | 30 +
.../server/upgrade/UpgradeCatalog300Test.java | 95 +-
ambari-server/src/test/python/TestAmbariServer.py | 600 +-
.../HIVE/test_jdbc_driver_config.py | 21 -
.../RANGER/test_db_flavor_config.py | 22 -
.../RANGER_KMS/test_db_flavor_config.py | 22 -
.../SPARK/2.2.0/test_service_advisor.py | 289 -
.../SQOOP/test_jdbc_driver_config.py | 22 -
.../python/stacks/2.0.6/OOZIE/test_oozie_server.py | 152 +-
.../python/stacks/2.1/FALCON/test_falcon_server.py | 6 +-
.../python/stacks/2.2/KNOX/test_knox_gateway.py | 29 +-
.../python/stacks/2.2/configs/oozie-upgrade.json | 39 +-
.../test/python/stacks/2.4/configs/default.json | 1 +
.../src/test/python/stacks/utils/RMFTestCase.py | 2 +
ambari-web/app/assets/test/tests.js | 5 +-
.../main/admin/kerberos/step1_controller.js | 2 +-
.../main/charts/heatmap_metrics/heatmap_metric.js | 158 +-
ambari-web/app/controllers/main/host/details.js | 563 +-
ambari-web/app/controllers/main/service/item.js | 2 +-
.../app/controllers/wizard/step3_controller.js | 2 +-
ambari-web/app/messages.js | 7 +-
.../app/mixins/common/widgets/widget_mixin.js | 12 +-
.../details/host_components/install_component.js | 6 +-
ambari-web/app/models/host_component.js | 3 +
ambari-web/app/styles/application.less | 17 +-
ambari-web/app/styles/bootstrap_overrides.less | 15 +-
ambari-web/app/styles/config_versions_control.less | 5 -
ambari-web/app/styles/dashboard.less | 14 +-
ambari-web/app/styles/hosts.less | 78 +-
ambari-web/app/styles/theme/bootstrap-ambari.css | 7 +
ambari-web/app/styles/wizard.less | 35 +-
.../templates/common/configs/service_config.hbs | 2 +-
.../common/configs/service_config_wizard.hbs | 2 +-
.../app/templates/main/dashboard/widgets.hbs | 2 +-
ambari-web/app/templates/main/host/details.hbs | 10 +-
.../templates/main/host/details/host_component.hbs | 289 +-
ambari-web/app/templates/main/host/summary.hbs | 121 +-
ambari-web/app/templates/wizard/step0.hbs | 2 +-
ambari-web/app/templates/wizard/step1.hbs | 2 -
ambari-web/app/templates/wizard/step3.hbs | 14 +-
.../step3/step3_host_warning_popup_footer.hbs | 2 +-
.../wizard/step3/step3_host_warnings_popup.hbs | 40 +-
ambari-web/app/templates/wizard/step4.hbs | 31 +-
ambari-web/app/templates/wizard/step8.hbs | 4 +-
ambari-web/app/utils/config.js | 2 +-
ambari-web/app/utils/date/date.js | 86 +-
ambari-web/app/utils/heatmap.js | 48 -
ambari-web/app/utils/number_utils.js | 5 +
.../app/views/common/widget/heatmap_widget_view.js | 24 +-
.../main/dashboard/widgets/hbase_master_uptime.js | 7 +-
.../main/dashboard/widgets/namenode_uptime.js | 7 +-
.../dashboard/widgets/resource_manager_uptime.js | 7 +-
.../main/dashboard/widgets/uptime_text_widget.js | 70 +-
.../views/main/host/details/host_component_view.js | 92 +-
ambari-web/app/views/main/host/summary.js | 170 +-
.../charts/heatmap_metrics/heatmap_metric_test.js | 260 +-
.../test/controllers/main/host/details_test.js | 620 +-
.../manage_config_groups_controller_test.js | 5 +-
ambari-web/test/mappers/service_mapper_test.js | 5 +-
.../add_component_config_initializer_test.js | 140 +
ambari-web/test/utils/date/date_test.js | 52 +-
ambari-web/test/utils/heatmap_test.js | 141 -
.../common/host_progress_popup_body_view_test.js | 33 +-
.../common/widget/heatmap_widget_view_test.js | 27 +-
.../stack_upgrade/upgrade_history_view_test.js | 4 +-
.../dashboard/widgets/hbase_master_uptime_test.js | 91 -
.../main/dashboard/widgets/namenode_uptime_test.js | 95 -
.../widgets/resource_manager_uptime_test.js | 83 -
.../dashboard/widgets/uptime_text_widget_test.js | 70 +-
.../main/host/details/host_component_view_test.js | 136 +-
ambari-web/test/views/main/host/summary_test.js | 396 +-
.../vendor/theme/fonts/Roboto-Bold-webfont.eot | Bin 0 -> 47517 bytes
.../vendor/theme/fonts/Roboto-Bold-webfont.svg | 607 ++
.../vendor/theme/fonts/Roboto-Bold-webfont.ttf | Bin 0 -> 47319 bytes
.../vendor/theme/fonts/Roboto-Bold-webfont.woff | Bin 0 -> 21320 bytes
.../main/resources/ui/app/routes/savedqueries.js | 10 +-
contrib/views/pom.xml | 5 +
docs/pom.xml | 14 +-
1012 files changed, 12578 insertions(+), 126923 deletions(-)
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/app.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/app.js
index 225eb12..98b0383 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/app.js
+++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/app.js
@@ -63,7 +63,7 @@ angular.module('ambariAdminConsole', [
};
}]);
- $httpProvider.responseInterceptors.push(['$rootScope', '$q', function (scope, $q) {
+ $httpProvider.interceptors.push(['$rootScope', '$q', function (scope, $q) {
function success(response) {
return response;
}
diff --git a/ambari-admin/src/main/resources/ui/admin-web/package.json b/ambari-admin/src/main/resources/ui/admin-web/package.json
index 40122e1..0d721d4 100644
--- a/ambari-admin/src/main/resources/ui/admin-web/package.json
+++ b/ambari-admin/src/main/resources/ui/admin-web/package.json
@@ -14,7 +14,7 @@
"gulp-flatten": "0.0.2",
"gulp-load-plugins": "0.5.0",
"gulp-order": "1.1.1",
- "gulp-plumber": "0.6.6",
+ "gulp-plumber": "1.1.0",
"gulp-size": "0.3.0",
"gulp-uglify": "0.2.1",
"gulp-useref": "0.4.2",
diff --git a/ambari-agent/conf/unix/ambari-agent b/ambari-agent/conf/unix/ambari-agent
index f010800..8168002 100755
--- a/ambari-agent/conf/unix/ambari-agent
+++ b/ambari-agent/conf/unix/ambari-agent
@@ -62,7 +62,7 @@ export PATH=/usr/sbin:/sbin:/usr/lib/ambari-server/*:$PATH
export AMBARI_CONF_DIR=$HOME_DIR/etc/ambari-server/conf:$PATH
# Because Ambari rpm unpacks modules here on all systems
-export PYTHONPATH=/usr/lib/python2.6/site-packages:${PYTHONPATH:-}
+export PYTHONPATH=/usr/lib/ambari-agent/lib:${PYTHONPATH:-}
export AMBARI_PID_DIR=`get_agent_property piddir`
export AMBARI_PID_DIR=`valid_path "${AMBARI_PID_DIR:?}"`
@@ -79,11 +79,11 @@ PYTHON_WRAP=/usr/bin/ambari-python-wrap
PIDFILE=$AMBARI_PID_DIR/$AMBARI_AGENT.pid
OUTFILE=$AMBARI_AGENT_LOG_DIR/ambari-agent.out
LOGFILE=$AMBARI_AGENT_LOG_DIR/ambari-agent.log
-AGENT_SCRIPT=/usr/lib/python2.6/site-packages/ambari_agent/main.py
+AGENT_SCRIPT=/usr/lib/ambari-agent/lib/ambari_agent/main.py
AGENT_TMP_DIR=/var/lib/ambari-agent/tmp
AGENT_WORKING_DIR=/var/lib/ambari-agent
-AMBARI_AGENT_PY_SCRIPT=/usr/lib/python2.6/site-packages/ambari_agent/AmbariAgent.py
-COMMON_DIR=/usr/lib/python2.6/site-packages/ambari_commons
+AMBARI_AGENT_PY_SCRIPT=/usr/lib/ambari-agent/lib/ambari_agent/AmbariAgent.py
+COMMON_DIR=/usr/lib/ambari-agent/lib/ambari_commons
COMMON_DIR_AGENT=/usr/lib/ambari-agent/lib/ambari_commons
OK=0
NOTOK=1
diff --git a/ambari-agent/conf/unix/ambari-env.sh b/ambari-agent/conf/unix/ambari-env.sh
index 1f93431..6458b87 100644
--- a/ambari-agent/conf/unix/ambari-env.sh
+++ b/ambari-agent/conf/unix/ambari-env.sh
@@ -17,7 +17,7 @@
# given through environment variable
AMBARI_PASSPHRASE="DEV"
export PATH=$PATH:/var/lib/ambari-agent
-export PYTHONPATH=$PYTHONPATH:/usr/lib/python2.6/site-packages
+export PYTHONPATH=/usr/lib/ambari-agent/lib:$PYTHONPATH
# customize python binary for ambari
# export PYTHON=/usr/bin/python2
diff --git a/ambari-agent/conf/unix/install-helper.sh b/ambari-agent/conf/unix/install-helper.sh
index 37bbe4a..e305866 100644
--- a/ambari-agent/conf/unix/install-helper.sh
+++ b/ambari-agent/conf/unix/install-helper.sh
@@ -18,22 +18,32 @@
# AGENT INSTALL HELPER #
##################################################################
-COMMON_DIR="/usr/lib/python2.6/site-packages/ambari_commons"
-RESOURCE_MANAGEMENT_DIR="/usr/lib/python2.6/site-packages/resource_management"
-JINJA_DIR="/usr/lib/python2.6/site-packages/ambari_jinja2"
-SIMPLEJSON_DIR="/usr/lib/python2.6/site-packages/ambari_simplejson"
-OLD_COMMON_DIR="/usr/lib/python2.6/site-packages/common_functions"
INSTALL_HELPER_SERVER="/var/lib/ambari-server/install-helper.sh"
COMMON_DIR_AGENT="/usr/lib/ambari-agent/lib/ambari_commons"
RESOURCE_MANAGEMENT_DIR_AGENT="/usr/lib/ambari-agent/lib/resource_management"
JINJA_AGENT_DIR="/usr/lib/ambari-agent/lib/ambari_jinja2"
SIMPLEJSON_AGENT_DIR="/usr/lib/ambari-agent/lib/ambari_simplejson"
-AMBARI_AGENT="/usr/lib/python2.6/site-packages/ambari_agent"
PYTHON_WRAPER_TARGET="/usr/bin/ambari-python-wrap"
AMBARI_AGENT_VAR="/var/lib/ambari-agent"
AMBARI_AGENT_BINARY="/etc/init.d/ambari-agent"
AMBARI_AGENT_BINARY_SYMLINK="/usr/sbin/ambari-agent"
+
+OLD_COMMON_DIR="/usr/lib/python2.6/site-packages/ambari_commons"
+OLD_RESOURCE_MANAGEMENT_DIR="/usr/lib/python2.6/site-packages/resource_management"
+OLD_JINJA_DIR="/usr/lib/python2.6/site-packages/ambari_jinja2"
+OLD_SIMPLEJSON_DIR="/usr/lib/python2.6/site-packages/ambari_simplejson"
+OLD_AMBARI_AGENT_DIR="/usr/lib/python2.6/site-packages/ambari_agent"
+
+COMMON_DIR="/usr/lib/ambari-agent/lib/ambari_commons"
+RESOURCE_MANAGEMENT_DIR="/usr/lib/ambari-agent/lib/resource_management"
+JINJA_DIR="/usr/lib/ambari-agent/lib/ambari_jinja2"
+SIMPLEJSON_DIR="/usr/lib/ambari-agent/lib/ambari_simplejson"
+STOMP_DIR="/usr/lib/ambari-agent/lib/ambari_stomp"
+WS4PY_DIR="/usr/lib/ambari-agent/lib/ambari_ws4py"
+OLD_COMMON_DIR="/usr/lib/ambari-agent/lib/common_functions"
+AMBARI_AGENT="/usr/lib/ambari-agent/lib/ambari_agent"
+
clean_pyc_files(){
# cleaning old *.pyc files
find ${RESOURCE_MANAGEMENT_DIR:?} -name *.pyc -exec rm {} \;
@@ -49,27 +59,8 @@ do_install(){
mv /etc/ambari-agent/conf.save /etc/ambari-agent/conf_$(date '+%d_%m_%y_%H_%M').save
fi
- # setting up /usr/sbin/ambari-agent symlink
- rm -f "$AMBARI_AGENT_BINARY_SYMLINK"
- ln -s "$AMBARI_AGENT_BINARY" "$AMBARI_AGENT_BINARY_SYMLINK"
-
- # setting ambari_commons shared resource
- rm -rf "$OLD_COMMON_DIR"
- if [ ! -d "$COMMON_DIR" ]; then
- ln -s "$COMMON_DIR_AGENT" "$COMMON_DIR"
- fi
- # setting resource_management shared resource
- if [ ! -d "$RESOURCE_MANAGEMENT_DIR" ]; then
- ln -s "$RESOURCE_MANAGEMENT_DIR_AGENT" "$RESOURCE_MANAGEMENT_DIR"
- fi
- # setting jinja2 shared resource
- if [ ! -d "$JINJA_DIR" ]; then
- ln -s "$JINJA_AGENT_DIR" "$JINJA_DIR"
- fi
- # setting simplejson shared resource
- if [ ! -d "$SIMPLEJSON_DIR" ]; then
- ln -s "$SIMPLEJSON_AGENT_DIR" "$SIMPLEJSON_DIR"
- fi
+ # these symlinks (or directories) where created in ambari releases prior to ambari-2.6.2. Do clean up.
+ rm -rf "$OLD_COMMON_DIR" "$OLD_RESOURCE_MANAGEMENT_DIR" "$OLD_JINJA_DIR" "$OLD_SIMPLEJSON_DIR" "$OLD_COMMON_DIR" "$OLD_AMBARI_AGENT_DIR"
# on nano Ubuntu, when umask=027 those folders are created without 'x' bit for 'others'.
# which causes failures when hadoop users try to access tmp_dir
@@ -124,6 +115,14 @@ do_install(){
mv $BAK ${BAK}_$(date '+%d_%m_%y_%H_%M').save
fi
fi
+
+ if [ -f "$AMBARI_ENV_RPMSAVE" ] ; then
+ PYTHON_PATH_LINE='export PYTHONPATH=/usr/lib/ambari-agent/lib:$PYTHONPATH'
+ grep "^$PYTHON_PATH_LINE\$" "$AMBARI_ENV_RPMSAVE" > /dev/null
+ if [ $? -ne 0 ] ; then
+ echo -e "\n$PYTHON_PATH_LINE" >> $AMBARI_ENV_RPMSAVE
+ fi
+ fi
}
do_remove(){
@@ -158,10 +157,6 @@ do_remove(){
rm -f $SIMPLEJSON_DIR
fi
- if [ -d "$OLD_COMMON_DIR" ]; then
- rm -f $OLD_COMMON_DIR
- fi
-
# if server package exists, restore their settings
if [ -f "$INSTALL_HELPER_SERVER" ]; then # call server shared files installer
$INSTALL_HELPER_SERVER install
diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml
index eede374..38a0ada 100644
--- a/ambari-agent/pom.xml
+++ b/ambari-agent/pom.xml
@@ -36,7 +36,7 @@
<package.log.dir>/var/log/ambari-agent</package.log.dir>
<package.pid.dir>/var/run/ambari-agent</package.pid.dir>
<skipTests>false</skipTests>
- <agent.install.dir>/usr/lib/python2.6/site-packages/ambari_agent</agent.install.dir>
+ <agent.install.dir>/usr/lib/ambari-agent/lib/ambari_agent</agent.install.dir>
<ambari_commons.install.dir>/usr/lib/ambari-agent/lib/ambari_commons</ambari_commons.install.dir>
<resource_management.install.dir>/usr/lib/ambari-agent/lib/resource_management</resource_management.install.dir>
<jinja.install.dir>/usr/lib/ambari-agent/lib/ambari_jinja2</jinja.install.dir>
@@ -285,17 +285,6 @@
</sources>
</mapping>
<mapping>
- <directory>/usr/lib/python2.6/site-packages</directory>
- <filemode>755</filemode>
- <username>root</username>
- <groupname>root</groupname>
- <sources>
- <source>
- <location>${project.build.directory}${dirsep}${project.artifactId}-${project.version}/usr/lib/python2.6/site-packages</location>
- </source>
- </sources>
- </mapping>
- <mapping>
<directory>/var/lib/ambari-agent</directory>
<filemode>755</filemode>
<username>root</username>
diff --git a/ambari-agent/src/main/package/rpm/posttrans_agent.sh b/ambari-agent/src/main/package/rpm/posttrans_agent.sh
index 723bb93..35ec59a 100644
--- a/ambari-agent/src/main/package/rpm/posttrans_agent.sh
+++ b/ambari-agent/src/main/package/rpm/posttrans_agent.sh
@@ -14,29 +14,9 @@
# limitations under the License
-RESOURCE_MANAGEMENT_DIR="/usr/lib/python2.6/site-packages/resource_management"
-RESOURCE_MANAGEMENT_DIR_AGENT="/usr/lib/ambari-agent/lib/resource_management"
-JINJA_DIR="/usr/lib/python2.6/site-packages/ambari_jinja2"
-JINJA_AGENT_DIR="/usr/lib/ambari-agent/lib/ambari_jinja2"
AMBARI_AGENT_BINARY="/etc/init.d/ambari-agent"
AMBARI_AGENT_BINARY_SYMLINK="/usr/sbin/ambari-agent"
-# remove RESOURCE_MANAGEMENT_DIR if it's a directory
-if [ -d "$RESOURCE_MANAGEMENT_DIR" ]; then # resource_management dir exists
- if [ ! -L "$RESOURCE_MANAGEMENT_DIR" ]; then # resource_management dir is not link
- rm -rf "$RESOURCE_MANAGEMENT_DIR"
- fi
-fi
-# setting resource_management shared resource
-if [ ! -d "$RESOURCE_MANAGEMENT_DIR" ]; then
- ln -s "$RESOURCE_MANAGEMENT_DIR_AGENT" "$RESOURCE_MANAGEMENT_DIR"
-fi
-
-# setting jinja2 shared resource
-if [ ! -d "$JINJA_DIR" ]; then
- ln -s "$JINJA_AGENT_DIR" "$JINJA_DIR"
-fi
-
# setting ambari-agent binary symlink
if [ ! -f "$AMBARI_AGENT_BINARY_SYMLINK" ]; then
ln -s "$AMBARI_AGENT_BINARY" "$AMBARI_AGENT_BINARY_SYMLINK"
diff --git a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
index da4dc4c..1a85e76 100644
--- a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
+++ b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
@@ -145,8 +145,8 @@ class ActionQueue(threading.Thread):
self.customServiceOrchestrator.cancel_command(task_id, reason)
def run(self):
- try:
- while not self.stopped():
+ while not self.stopped():
+ try:
self.processBackgroundQueueSafeEmpty()
self.controller.get_status_commands_executor().process_results() # process status commands
try:
@@ -177,9 +177,8 @@ class ActionQueue(threading.Thread):
pass
except (Queue.Empty):
pass
- except:
- logger.exception("ActionQueue thread failed with exception:")
- raise
+ except:
+ logger.exception("ActionQueue thread failed with exception. Re-running it")
logger.info("ActionQueue thread has successfully finished")
diff --git a/ambari-agent/src/main/python/ambari_agent/AmbariAgent.py b/ambari-agent/src/main/python/ambari_agent/AmbariAgent.py
index aeb200c..8475551 100644
--- a/ambari-agent/src/main/python/ambari_agent/AmbariAgent.py
+++ b/ambari-agent/src/main/python/ambari_agent/AmbariAgent.py
@@ -27,7 +27,7 @@ from Controller import AGENT_AUTO_RESTART_EXIT_CODE
if os.environ.has_key("PYTHON_BIN"):
AGENT_SCRIPT = os.path.join(os.environ["PYTHON_BIN"],"site-packages/ambari_agent/main.py")
else:
- AGENT_SCRIPT = "/usr/lib/python2.6/site-packages/ambari_agent/main.py"
+ AGENT_SCRIPT = "/usr/lib/ambari-agent/lib/ambari_agent/main.py"
if os.environ.has_key("AMBARI_PID_DIR"):
AGENT_PID_FILE = os.path.join(os.environ["AMBARI_PID_DIR"],"ambari-agent.pid")
else:
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
index 7ee3b1c..3db90ea 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
@@ -20,7 +20,7 @@ limitations under the License.
# For compatibility with different OSes
# Edit PYTHONPATH to be able to import common_functions
import sys
-sys.path.append("/usr/lib/python2.6/site-packages/")
+sys.path.append("/usr/lib/ambari-agent/lib/")
########################################################
import os
diff --git a/ambari-agent/src/test/python/ambari_agent/TestProcessUtils.py b/ambari-agent/src/test/python/ambari_agent/TestProcessUtils.py
deleted file mode 100644
index 8331910..0000000
--- a/ambari-agent/src/test/python/ambari_agent/TestProcessUtils.py
+++ /dev/null
@@ -1,224 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-from ambari_agent import main
-
-main.MEMORY_LEAK_DEBUG_FILEPATH = "/tmp/memory_leak_debug.out"
-import unittest
-import signal
-import subprocess, time
-from mock.mock import patch, MagicMock, PropertyMock, call
-from ambari_commons import process_utils
-
-process_tree = {"111": "222\n 22",
- "222": "333\n 33",
- "22": "44\n 444",}
-
-
-class TestProcessUtils(unittest.TestCase):
- @patch("subprocess.Popen")
- def test_kill(self, popen_mock):
- process_mock = MagicMock()
- process_mock.communicate.return_value = (None, None)
- returncode_mock = PropertyMock()
- returncode_mock.return_value = 0
- type(process_mock).returncode = returncode_mock
- popen_mock.return_value = process_mock
- process_utils.kill_pids(["12321113230", "2312415453"], signal.SIGTERM)
- expected = [call(['kill', '-15', '12321113230', '2312415453'], stderr=-1, stdout=-1)]
- self.assertEquals(popen_mock.call_args_list, expected)
-
- @patch("subprocess.Popen")
- def test_get_children(self, popen_mock):
-
- process_mock = MagicMock()
- process_mock.communicate.return_value = ("123 \n \n 321\n", None)
- popen_mock.return_value = process_mock
- returncode_mock = PropertyMock()
- returncode_mock.return_value = 0
- type(process_mock).returncode = returncode_mock
- result = process_utils.get_children("2312415453")
-
- self.assertEquals(result, ["123", "321"])
-
- expected = [
- call(['ps', '-o', 'pid', '--no-headers', '--ppid', '2312415453'], stderr=subprocess.PIPE, stdout=subprocess.PIPE)]
- self.assertEquals(popen_mock.call_args_list, expected)
-
- @patch("subprocess.Popen")
- def test_get_flat_process_tree(self, popen_mock):
- def side_effect(*args, **kwargs):
- process_mock = MagicMock()
- returncode_mock = PropertyMock()
- returncode_mock.return_value = 0
- type(process_mock).returncode = returncode_mock
- if args[0][5] in process_tree.keys():
- process_mock.communicate.return_value = (process_tree[args[0][5]], None)
- else:
- process_mock.communicate.return_value = ("", None)
- return process_mock
-
- popen_mock.side_effect = side_effect
- result = process_utils.get_flat_process_tree("111")
- self.assertEquals(result, ['111', '222', '333', '33', '22', '44', '444'])
-
- expected = [call(['ps', '-o', 'pid', '--no-headers', '--ppid', '111'], stderr=-1, stdout=-1),
- call(['ps', '-o', 'pid', '--no-headers', '--ppid', '222'], stderr=-1, stdout=-1),
- call(['ps', '-o', 'pid', '--no-headers', '--ppid', '333'], stderr=-1, stdout=-1),
- call(['ps', '-o', 'pid', '--no-headers', '--ppid', '33'], stderr=-1, stdout=-1),
- call(['ps', '-o', 'pid', '--no-headers', '--ppid', '22'], stderr=-1, stdout=-1),
- call(['ps', '-o', 'pid', '--no-headers', '--ppid', '44'], stderr=-1, stdout=-1),
- call(['ps', '-o', 'pid', '--no-headers', '--ppid', '444'], stderr=-1, stdout=-1)]
- self.assertEquals(popen_mock.call_args_list, expected)
-
- @patch("subprocess.Popen")
- def test_get_command_by_pid(self, popen_mock):
-
- process_mock = MagicMock()
- process_mock.communicate.return_value = ("yum something", None)
- returncode_mock = PropertyMock()
- returncode_mock.return_value = 0
- type(process_mock).returncode = returncode_mock
- popen_mock.return_value = process_mock
-
- result = process_utils.get_command_by_pid("2312415453")
-
- self.assertEquals(result, "yum something")
-
- expected = [call(['ps', '-p', '2312415453', '-o', 'command', '--no-headers'], stderr=-1, stdout=-1)]
- self.assertEquals(popen_mock.call_args_list, expected)
-
- @patch("subprocess.Popen")
- def test_get_command_by_pid_not_exist(self, popen_mock):
-
- process_mock = MagicMock()
- process_mock.communicate.return_value = ("", None)
- returncode_mock = PropertyMock()
- returncode_mock.return_value = 1
- type(process_mock).returncode = returncode_mock
- popen_mock.return_value = process_mock
-
- result = process_utils.get_command_by_pid("2312415453")
-
- self.assertEquals(result, "NOT_FOUND[2312415453]")
-
- expected = [call(['ps', '-p', '2312415453', '-o', 'command', '--no-headers'], stderr=-1, stdout=-1)]
- self.assertEquals(popen_mock.call_args_list, expected)
-
- @patch("subprocess.Popen")
- def test_is_process_running(self, popen_mock):
-
- process_mock = MagicMock()
- process_mock.communicate.return_value = ("2312415453", None)
- returncode_mock = PropertyMock()
- returncode_mock.return_value = 0
- type(process_mock).returncode = returncode_mock
- popen_mock.return_value = process_mock
-
- result = process_utils.is_process_running("2312415453")
-
- self.assertEquals(result, True)
-
- expected = [call(['ps', '-p', '2312415453', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1)]
- self.assertEquals(popen_mock.call_args_list, expected)
-
- @patch("subprocess.Popen")
- def test_is_process_not_running(self, popen_mock):
-
- process_mock = MagicMock()
- process_mock.communicate.return_value = ("", None)
- returncode_mock = PropertyMock()
- returncode_mock.return_value = 1
- type(process_mock).returncode = returncode_mock
- popen_mock.return_value = process_mock
-
- result = process_utils.is_process_running("2312415453")
-
- self.assertEquals(result, False)
-
- expected = [call(['ps', '-p', '2312415453', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1)]
- self.assertEquals(popen_mock.call_args_list, expected)
-
- @patch("subprocess.Popen")
- def test_get_processes_running(self, popen_mock):
- def side_effect(*args, **kwargs):
- process_mock = MagicMock()
- returncode_mock = PropertyMock()
- if args[0][2] == "4321":
- returncode_mock.return_value = 0
- process_mock.communicate.return_value = ("4321", None)
- else:
- returncode_mock.return_value = 1
- process_mock.communicate.return_value = (None, None)
- type(process_mock).returncode = returncode_mock
- return process_mock
-
- popen_mock.side_effect = side_effect
-
- result = process_utils.get_processes_running(["1234", "4321"])
-
- self.assertEquals(result, ["4321"])
-
- expected = [call(['ps', '-p', '1234', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
- call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1)]
- self.assertEquals(popen_mock.call_args_list, expected)
-
- @patch("time.sleep")
- @patch("subprocess.Popen")
- def test_wait_for_process_death(self, popen_mock, sleep_mock):
-
- process_mock = MagicMock()
- process_mock.communicate.side_effect = [("4321", None),("4321", None),(None, None)]
- returncode_mock = PropertyMock()
- returncode_mock.side_effect = [0, 0, 1]
- type(process_mock).returncode = returncode_mock
- popen_mock.return_value = process_mock
-
- process_utils.wait_for_process_death("4321")
-
- expected = [call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
- call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
- call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1)]
- self.assertEquals(popen_mock.call_args_list, expected)
- expected = [call(0.1), call(0.1)]
- self.assertEquals(sleep_mock.call_args_list, expected)
-
- @patch("time.sleep")
- @patch("subprocess.Popen")
- def test_wait_for_entire_process_tree_death(self, popen_mock, sleep_mock):
-
- process_mock = MagicMock()
- process_mock.communicate.side_effect = [("1234", None), (None, None), ("4321", None), ("4321", None), (None, None)]
- returncode_mock = PropertyMock()
- returncode_mock.side_effect = [0, 1, 0, 0, 1]
- type(process_mock).returncode = returncode_mock
- popen_mock.return_value = process_mock
-
- process_utils.wait_for_entire_process_tree_death(["1234", "4321"])
-
- expected = [call(['ps', '-p', '1234', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
- call(['ps', '-p', '1234', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
- call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
- call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1),
- call(['ps', '-p', '4321', '-o', 'pid', '--no-headers'], stderr=-1, stdout=-1)]
- self.assertEquals(popen_mock.call_args_list, expected)
- expected = [call(0.1), call(0.1), call(0.1)]
- self.assertEquals(sleep_mock.call_args_list, expected)
diff --git a/ambari-agent/src/test/python/ambari_agent/TestShell.py b/ambari-agent/src/test/python/ambari_agent/TestShell.py
index 47923bd..0f72020 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestShell.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestShell.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-'''
+"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@@ -9,71 +9,138 @@ regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
-
http://www.apache.org/licenses/LICENSE-2.0
-
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-'''
+"""
+from contextlib import contextmanager
-from ambari_agent import main
-main.MEMORY_LEAK_DEBUG_FILEPATH = "/tmp/memory_leak_debug.out"
-import os
import unittest
-import tempfile
+import signal
from mock.mock import patch, MagicMock, call
-from ambari_agent.AmbariConfig import AmbariConfig
from ambari_commons import shell
-from ambari_commons.shell import shellRunner
-from sys import platform as _platform
-from only_for_platform import not_for_platform, PLATFORM_WINDOWS
-import subprocess, time
+from ambari_commons import OSCheck
+from StringIO import StringIO
+
+ROOT_PID = 10
+ROOT_PID_CHILDREN = [10, 11, 12, 13]
+shell.logger = MagicMock() # suppress any log output
+
+__proc_fs = {
+ "/proc/10/task/10/children": "11 12",
+ "/proc/10/comm": "a",
+ "/proc/10/cmdline": "",
+
+ "/proc/11/task/11/children": "13",
+ "/proc/11/comm": "b",
+ "/proc/11/cmdline": "",
+
+ "/proc/12/task/12/children": "",
+ "/proc/12/comm": "c",
+ "/proc/12/cmdline": "",
+
+ "/proc/13/task/13/children": "",
+ "/proc/13/comm": "d",
+ "/proc/13/cmdline": ""
+}
+
+__proc_fs_yum = {
+ "/proc/10/task/10/children": "11",
+ "/proc/10/comm": "a",
+ "/proc/10/cmdline": "",
+
+ "/proc/11/task/11/children": "",
+ "/proc/11/comm": "yum",
+ "/proc/11/cmdline": "yum install something"
+}
+
+# Remove any wait delay, no need for tests
+__old_waiter = shell.wait_for_process_list_kill
+
+
+def __wait_for_process_list_kill(pids, timeout=5, check_step_time=0.1):
+ return __old_waiter(pids, 0, check_step_time)
+
+
+shell.wait_for_process_list_kill = __wait_for_process_list_kill
+
+
+class FakeSignals(object):
+ SIGTERM = signal.SIG_IGN
+ SIGKILL = signal.SIG_IGN
+
+
+@contextmanager
+def _open_mock(path, open_mode):
+ if path in __proc_fs:
+ yield StringIO(__proc_fs[path])
+ else:
+ yield StringIO("")
+
+
+@contextmanager
+def _open_mock_yum(path, open_mode):
+ if path in __proc_fs:
+ yield StringIO(__proc_fs_yum[path])
+ else:
+ yield StringIO("")
+
-@not_for_platform(PLATFORM_WINDOWS)
class TestShell(unittest.TestCase):
+ @patch("__builtin__.open", new=MagicMock(side_effect=_open_mock))
+ def test_get_all_children(self):
+
+ pid_list = [item[0] for item in shell.get_all_children(ROOT_PID)]
+
+ self.assertEquals(len(ROOT_PID_CHILDREN), len(pid_list))
+ self.assertEquals(ROOT_PID, pid_list[0])
+
+ for i in ROOT_PID_CHILDREN:
+ self.assertEquals(True, i in pid_list)
+
+ @patch("__builtin__.open", new=MagicMock(side_effect=_open_mock))
+ @patch.object(OSCheck, "get_os_family", new=MagicMock(return_value="redhat"))
+ @patch.object(shell, "signal", new_callable=FakeSignals)
+ @patch("os.listdir")
+ @patch("os.kill")
+ def test_kill_process_with_children(self, os_kill_mock, os_list_dir_mock, fake_signals):
+ pid_list = [item[0] for item in shell.get_all_children(ROOT_PID)]
+ pid_list_str = [str(i) for i in ROOT_PID_CHILDREN]
+ reverse_pid_list = sorted(pid_list, reverse=True)
+ os_list_dir_mock.side_effect = [pid_list_str, [], [], []]
+
+ shell.kill_process_with_children(ROOT_PID)
+
+ # test pid kill by SIGTERM
+ os_kill_pids = [item[0][0] for item in os_kill_mock.call_args_list]
+ self.assertEquals(len(os_kill_pids), len(pid_list))
+ self.assertEquals(reverse_pid_list, os_kill_pids)
+
+ os_kill_mock.reset_mock()
+ os_list_dir_mock.reset_mock()
+
+ os_list_dir_mock.side_effect = [pid_list_str, pid_list_str, pid_list_str, pid_list_str, [], []]
+ shell.kill_process_with_children(ROOT_PID)
+
+ # test pid kill by SIGKILL
+ os_kill_pids = [item[0][0] for item in os_kill_mock.call_args_list]
+ self.assertEquals(len(os_kill_pids), len(pid_list)*2)
+ self.assertEquals(reverse_pid_list + reverse_pid_list, os_kill_pids)
+
+ @patch("__builtin__.open", new=MagicMock(side_effect=_open_mock_yum))
+ @patch.object(OSCheck, "get_os_family", new=MagicMock(return_value="redhat"))
+ @patch.object(shell, "signal", new_callable=FakeSignals)
+ @patch("os.listdir")
+ @patch("os.kill")
+ def test_kill_process_with_children_except_yum(self, os_kill_mock, os_list_dir_mock, fake_signals):
+ os_list_dir_mock.side_effect = [["10", "12", "20"], [], [], []]
+ shell.kill_process_with_children(ROOT_PID)
- @patch("os.setuid")
- def test_changeUid(self, os_setUIDMock):
- shell.threadLocal.uid = 9999
- shell.changeUid()
- self.assertTrue(os_setUIDMock.called)
-
-
- @patch("pwd.getpwnam")
- def test_shellRunner_run(self, getpwnamMock):
- sh = shellRunner()
- result = sh.run(['echo'])
- self.assertEquals(result['exitCode'], 0)
- self.assertEquals(result['error'], '')
-
- getpwnamMock.return_value = [os.getuid(), os.getuid(), os.getuid()]
- result = sh.run(['echo'], 'non_exist_user_name')
- self.assertEquals(result['exitCode'], 0)
- self.assertEquals(result['error'], '')
-
- def test_kill_process_with_children(self):
- if _platform == "linux" or _platform == "linux2": # Test is Linux-specific
- sleep_cmd = "sleep 314"
- test_cmd = """ (({0}) & ({0} & {0})) """.format(sleep_cmd)
- # Starting process tree (multiple process groups)
- test_process = subprocess.Popen(test_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
- time.sleep(0.3) # Delay to allow subprocess to start
- # Check if processes are running
- ps_cmd = """ps auxww """
- ps_process = subprocess.Popen(ps_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
- (out, err) = ps_process.communicate()
- self.assertTrue(sleep_cmd in out)
- # Kill test process
- shell.kill_process_with_children(test_process.pid)
- test_process.communicate()
- # Now test process should not be running
- ps_process = subprocess.Popen(ps_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
- (out, err) = ps_process.communicate()
- self.assertFalse(sleep_cmd in out)
- else:
- # Do not run under other systems
- pass
+ # test clean pid by SIGTERM
+ os_kill_pids = [item[0][0] for item in os_kill_mock.call_args_list]
+ self.assertEquals(len(os_kill_pids), 1)
+ self.assertEquals([10], os_kill_pids)
diff --git a/ambari-common/src/main/python/ambari_commons/process_utils.py b/ambari-common/src/main/python/ambari_commons/process_utils.py
deleted file mode 100644
index 50ffd02..0000000
--- a/ambari-common/src/main/python/ambari_commons/process_utils.py
+++ /dev/null
@@ -1,100 +0,0 @@
-# !/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import subprocess
-import time
-
-check_time_delay = 0.1 # seconds between checks of process killed
-
-
-def get_children(pid):
- PSCMD = ["ps", "-o", "pid", "--no-headers", "--ppid", str(pid)]
- ps_process = subprocess.Popen(PSCMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- stdout, stderr = ps_process.communicate()
- if ps_process.returncode != 0:
- return []
- return stdout.split()
-
-
-def get_flat_process_tree(pid):
- """
- :param pid: process id of parent process
- :return: list of child process pids. Resulting list also includes parent pid
- """
- res = [str(pid)]
- children = get_children(pid)
- for child in children:
- res += get_flat_process_tree(child)
- return res
-
-
-def kill_pids(pids, signal):
- from resource_management.core.exceptions import Fail
- CMD = ["kill", "-" + str(signal)]
- CMD.extend(pids)
- process = subprocess.Popen(CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- stdout, stderr = process.communicate()
- if process.returncode != 0:
- raise Fail("Unable to kill PIDs {0} : {1}".format(str(pids),stderr))
-
-
-def get_command_by_pid(pid):
- CMD = ["ps", "-p", str(pid), "-o", "command", "--no-headers"]
- process = subprocess.Popen(CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- stdout, stderr = process.communicate()
- if process.returncode != 0:
- return "NOT_FOUND[%s]" % pid
- return stdout
-
-
-def wait_for_entire_process_tree_death(pids):
- for child in pids:
- wait_for_process_death(child)
-
-
-def wait_for_process_death(pid, timeout=5):
- start = time.time()
- current_time = start
- while is_process_running(pid) and current_time < start + timeout:
- time.sleep(check_time_delay)
- current_time = time.time()
-
-
-def is_process_running(pid):
- CMD = ["ps", "-p", str(pid), "-o", "pid", "--no-headers"]
- process = subprocess.Popen(CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- stdout, stderr = process.communicate()
- if process.returncode != 0:
- return False
- return pid in stdout
-
-
-
-def get_processes_running(process_pids):
- """
- Checks what processes are still running
- :param process_pids: list of process pids
- :return: list of pids for processes that are still running
- """
- result = []
- for pid in process_pids:
- if is_process_running(pid):
- result.append(pid)
- return result
diff --git a/ambari-common/src/main/python/ambari_commons/shell.py b/ambari-common/src/main/python/ambari_commons/shell.py
index 5b29de0..33756da 100644
--- a/ambari-common/src/main/python/ambari_commons/shell.py
+++ b/ambari-common/src/main/python/ambari_commons/shell.py
@@ -27,8 +27,7 @@ from contextlib import contextmanager
from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyImpl, OsFamilyFuncImpl
-from ambari_commons.process_utils import get_flat_process_tree, kill_pids, wait_for_entire_process_tree_death, \
- get_processes_running, get_command_by_pid
+from resource_management.core import sudo
logger = logging.getLogger()
@@ -231,35 +230,170 @@ class shellRunnerWindows(shellRunner):
return _dict_to_object({'exitCode': code, 'output': out, 'error': err})
-#linux specific code
-@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-def kill_process_with_children(parent_pid):
+def get_all_children(base_pid):
"""
- Kills process tree starting from a given pid.
- :param parent_pid: head of tree
- :param graceful_kill_delays: map <command name, custom delay between SIGTERM and SIGKILL>
- :return:
+ Return all child PIDs of base_pid process
+
+ :param base_pid starting PID to scan for children
+ :return tuple of the following: pid, binary name, command line incl. binary
+
+ :type base_pid int
+ :rtype list[(int, str, str)]
"""
+ parent_pid_path_pattern = "/proc/{0}/task/{0}/children"
+ comm_path_pattern = "/proc/{0}/comm"
+ cmdline_path_pattern = "/proc/{0}/cmdline"
+
+ def read_children(pid):
+ try:
+ with open(parent_pid_path_pattern.format(pid), "r") as f:
+ return [int(item) for item in f.readline().strip().split(" ")]
+ except (IOError, ValueError):
+ return []
+
+ def read_command(pid):
+ try:
+ with open(comm_path_pattern.format(pid), "r") as f:
+ return f.readline().strip()
+ except IOError:
+ return ""
+
+ def read_cmdline(pid):
+ try:
+ with open(cmdline_path_pattern.format(pid), "r") as f:
+ return f.readline().strip()
+ except IOError:
+ return ""
+
+ pids = []
+ scan_pending = [int(base_pid)]
+
+ while scan_pending:
+ curr_pid = scan_pending.pop(0)
+ children = read_children(curr_pid)
- pids = get_flat_process_tree(parent_pid)
+ pids.append((curr_pid, read_command(curr_pid), read_cmdline(curr_pid)))
+ scan_pending.extend(children)
+
+ return pids
+
+
+def is_pid_exists(pid):
+ """
+ Check if process with PID still exist (not counting it real state)
+
+ :type pid int
+ :rtype bool
+ """
+ pid_path = "/proc/{0}"
try:
- kill_pids(pids, signal.SIGTERM)
- except Exception, e:
- logger.warn("Failed to kill PID %d" % parent_pid)
- logger.warn("Reported error: " + repr(e))
+ return os.path.exists(pid_path.format(pid))
+ except (OSError, IOError):
+ logger.debug("Failed to check PID existence")
+ return False
+
+
+def get_existing_pids(pids):
+ """
+ Check if process with pid still exists (not counting it real state).
+
+ Optimized to check PID list at once.
+
+ :param pids list of PIDs to filter
+ :return list of still existing PID
- wait_for_entire_process_tree_death(pids)
+ :type pids list[int]
+ :rtype list[int]
+ """
+
+ existing_pid_list = []
try:
- running_processes = get_processes_running(pids)
- if running_processes:
- process_names = map(lambda x: get_command_by_pid(x), running_processes)
- logger.warn("These PIDs %s did not die after SIGTERM, sending SIGKILL. Exact commands to be killed:\n %s" %
- (", ".join(running_processes), "\n".join(process_names)))
- kill_pids(running_processes, signal.SIGKILL)
- except Exception, e:
- logger.error("Failed to send SIGKILL to PID %d. Process exited?" % parent_pid)
- logger.error("Reported error: " + repr(e))
+ all_existing_pid_list = [int(item) for item in os.listdir("/proc") if item.isdigit()]
+ except (OSError, IOError):
+ logger.debug("Failed to check PIDs existence")
+ return existing_pid_list
+
+ for pid_item in pids:
+ if pid_item in all_existing_pid_list:
+ existing_pid_list.append(pid_item)
+
+ return existing_pid_list
+
+
+def wait_for_process_list_kill(pids, timeout=5, check_step_time=0.1):
+ """
+ Process tree waiter
+
+ :type pids list[int]
+ :type timeout int|float
+ :type check_step_time int|float
+
+ :param pids list of PIDs to watch
+ :param timeout how long wait till giving up, seconds. Set 0 for nowait or None for infinite time
+ :param check_step_time how often scan for existing PIDs, seconds
+ """
+ from threading import Thread, Event
+ import time
+
+ stop_waiting = Event()
+
+ def _wait_loop():
+ while not stop_waiting.is_set() and get_existing_pids(pids):
+ time.sleep(check_step_time)
+
+ if timeout == 0: # no need for loop if no timeout is set
+ return
+
+ th = Thread(target=_wait_loop)
+ stop_waiting.clear()
+
+ th.start()
+ th.join(timeout=timeout)
+ stop_waiting.set()
+
+ th.join()
+
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def kill_process_with_children(base_pid):
+ """
+ Process tree killer
+
+ :type base_pid int
+ """
+ exception_list = ["apt-get", "apt", "yum", "zypper", "zypp"]
+ signals_to_post = {
+ "SIGTERM": signal.SIGTERM,
+ "SIGKILL": signal.SIGKILL
+ }
+ full_child_pids = get_all_children(base_pid)
+ all_child_pids = [item[0] for item in full_child_pids if item[1].lower() not in exception_list and item[0] != os.getpid()]
+ error_log = []
+
+ for sig_name, sig in signals_to_post.items():
+ # we need to kill processes from the bottom of the tree
+ pids_to_kill = sorted(get_existing_pids(all_child_pids), reverse=True)
+ for pid in pids_to_kill:
+ try:
+ sudo.kill(pid, sig)
+ except OSError as e:
+ error_log.append((sig_name, pid, repr(e)))
+
+ if pids_to_kill:
+ wait_for_process_list_kill(pids_to_kill)
+ still_existing_pids = get_existing_pids(pids_to_kill)
+ if still_existing_pids:
+ logger.warn("These PIDs {0} did not respond to {1} signal. Detailed commands list:\n {2}".format(
+ ", ".join([str(i) for i in still_existing_pids]),
+ sig_name,
+ "\n".join([i[2] for i in full_child_pids if i[0] in still_existing_pids])
+ ))
+
+ if get_existing_pids(all_child_pids) and error_log: # we're unable to kill all requested PIDs
+ logger.warn("Process termination error log:\n")
+ for error_item in error_log:
+ logger.warn("PID: {0}, Process: {1}, Exception message: {2}".format(*error_item))
def _changeUid():
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
index 68ee607..c505969 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
@@ -54,6 +54,9 @@ def get_lzo_packages():
return lzo_packages
+def is_gpl_license_accepted():
+ return default("/hostLevelParams/gpl_license_accepted", False)
+
def should_install_lzo():
"""
Return true if lzo is enabled via core-site.xml and GPL license (required for lzo) is accepted.
@@ -65,8 +68,7 @@ def should_install_lzo():
if not lzo_enabled:
return False
- is_gpl_license_accepted = default("/hostLevelParams/gpl_license_accepted", False)
- if not is_gpl_license_accepted:
+ if not is_gpl_license_accepted():
Logger.warning(INSTALLING_LZO_WITHOUT_GPL)
return False
@@ -79,6 +81,9 @@ def install_lzo_if_needed():
if not should_install_lzo():
return
+ # If user has just accepted GPL license. GPL repository can not yet be present.
+ Script.repository_util.create_repo_files()
+
lzo_packages = get_lzo_packages()
config = Script.get_config()
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
index 8a2ff25..7a2635f 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
@@ -239,11 +239,13 @@ def get_nameservice(hdfs_site):
"""
name_service = hdfs_site.get('dfs.internal.nameservices', None)
if not name_service:
- import re
name_service = hdfs_site.get('dfs.nameservices', None)
- if name_service:
- for ns in name_service.split(","):
- if 'dfs.namenode.shared.edits.dir' in hdfs_site and re.match(r'.*%s$' % ns, hdfs_site['dfs.namenode.shared.edits.dir']): # better would be core_site['fs.defaultFS'] but it's not available
- return ns
- return name_service.split(",")[0] # default to return the first nameservice
+
+ if name_service and ',' in name_service:
+ import re
+ for ns in name_service.split(","):
+ if 'dfs.namenode.shared.edits.dir' in hdfs_site and re.match(r'.*%s$' % ns, hdfs_site['dfs.namenode.shared.edits.dir']): # better would be core_site['fs.defaultFS'] but it's not available
+ return ns
+ return name_service.split(",")[0] # default to return the first nameservice
+
return name_service
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
index ef54f40..7faa33f 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
@@ -17,61 +17,83 @@ limitations under the License.
"""
+from ambari_commons.os_check import OSCheck
from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
from resource_management.libraries.resources.repository import Repository
+from resource_management.libraries.functions.is_empty import is_empty
import ambari_simplejson as json
-__all__ = ["create_repo_files", "CommandRepository"]
+__all__ = ["RepositoryUtil", "CommandRepository"]
# components_lits = repoName + postfix
UBUNTU_REPO_COMPONENTS_POSTFIX = "main"
+class RepositoryUtil:
+ def __init__(self, config, tags_to_skip):
+ self.tags_to_skip = tags_to_skip
-def create_repo_files(template, command_repository):
- """
- Creates repositories in a consistent manner for all types
- :param command_repository: a CommandRepository instance
- :type command_repository CommandRepository
- :return: a dictionary with repo ID => repo file name mapping
- """
+ # repo templates
+ repo_file = config['repositoryFile']
+ repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
+ repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
- if command_repository.version_id is None:
- raise Fail("The command repository was not parsed correctly")
+ if is_empty(repo_file):
+ return
- if 0 == len(command_repository.items):
- Logger.warning(
- "Repository for {0}/{1} has no repositories. Ambari may not be managing this version.".format(
- command_repository.stack_name, command_repository.version_string))
- return {}
+ self.template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu
+ self.command_repository = CommandRepository(repo_file)
- append_to_file = False # initialize to False to create the file anew.
- repo_files = {}
-
- for repository in command_repository.items:
-
- if repository.repo_id is None:
- raise Fail("Repository with url {0} has no id".format(repository.base_url))
+ def create_repo_files(self):
+ """
+ Creates repositories in a consistent manner for all types
+ :return: a dictionary with repo ID => repo file name mapping
+ """
+ if self.command_repository.version_id is None:
+ raise Fail("The command repository was not parsed correctly")
- if not repository.ambari_managed:
+ if 0 == len(self.command_repository.items):
Logger.warning(
- "Repository for {0}/{1}/{2} is not managed by Ambari".format(
- command_repository.stack_name, command_repository.version_string, repository.repo_id))
- else:
- Repository(repository.repo_id,
- action="create",
- base_url=repository.base_url,
- mirror_list=repository.mirrors_list,
- repo_file_name=command_repository.repo_filename,
- repo_template=template,
- components=repository.ubuntu_components,
- append_to_file=append_to_file)
- append_to_file = True
- repo_files[repository.repo_id] = command_repository.repo_filename
-
- return repo_files
+ "Repository for {0}/{1} has no repositories. Ambari may not be managing this version.".format(
+ self.command_repository.stack_name, self.command_repository.version_string))
+ return {}
+
+ append_to_file = False # initialize to False to create the file anew.
+ repo_files = {}
+ for repository in self.command_repository.items:
+ if repository.repo_id is None:
+ raise Fail("Repository with url {0} has no id".format(repository.base_url))
+
+ if self.tags_to_skip & repository.tags:
+ Logger.info("Repository with url {0} is not created due to its tags: {1}".format(repository.base_url, repository.tags))
+ continue
+
+ if not repository.ambari_managed:
+ Logger.warning(
+ "Repository for {0}/{1}/{2} is not managed by Ambari".format(
+ self.command_repository.stack_name, self.command_repository.version_string, repository.repo_id))
+ else:
+ Repository(repository.repo_id,
+ action="create",
+ base_url=repository.base_url,
+ mirror_list=repository.mirrors_list,
+ repo_file_name=self.command_repository.repo_filename,
+ repo_template=self.template,
+ components=repository.ubuntu_components,
+ append_to_file=append_to_file)
+ append_to_file = True
+ repo_files[repository.repo_id] = self.command_repository.repo_filename
+
+ return repo_files
+def create_repo_files(template, command_repository):
+ """
+ DEPRECATED. Is present for usage by old mpacks.
+ Please use Script.repository_util.create_repo_files() instead.
+ """
+ from resource_management.libraries.script import Script
+ return RepositoryUtil(Script.get_config(), set()).create_repo_files()
def _find_value(dictionary, key, default=None):
"""
@@ -146,6 +168,7 @@ class CommandRepositoryItem(object):
self.components = _find_value(json_dict, 'components')
self.base_url = _find_value(json_dict, 'baseUrl')
self.mirrors_list = _find_value(json_dict, 'mirrorsList')
+ self.tags = set(_find_value(json_dict, 'tags', default=[]))
self.ambari_managed = _find_value(json_dict, 'ambariManaged', default=True)
self.ubuntu_components = [self.distribution if self.distribution else self.repo_name] + \
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py
index 12356ed..b92c77e 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py
@@ -32,9 +32,11 @@ from resource_management.core.logger import Logger
__all__ = ["upload_configuration_to_zk", "create_collection", "setup_kerberos", "set_cluster_prop",
"setup_kerberos_plugin", "create_znode", "check_znode", "secure_solr_znode", "secure_znode"]
-def __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, separated_znode=False):
+def __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts=None, separated_znode=False):
sudo = AMBARI_SUDO_BINARY
- solr_cli_prefix = format('{sudo} JAVA_HOME={java64_home} /usr/lib/ambari-infra-solr-client/solrCloudCli.sh ' \
+
+ infra_solr_cli_opts= format(' INFRA_SOLR_CLI_OPTS="{java_opts}"') if java_opts is not None else ''
+ solr_cli_prefix = format('{sudo} JAVA_HOME={java64_home}{infra_solr_cli_opts} /usr/lib/ambari-infra-solr-client/solrCloudCli.sh ' \
'--zookeeper-connect-string {zookeeper_quorum}')
if separated_znode:
solr_cli_prefix+=format(' --znode {solr_znode}')
@@ -49,7 +51,7 @@ def __append_flags_if_exists(command, flagsDict):
return command
def upload_configuration_to_zk(zookeeper_quorum, solr_znode, config_set, config_set_dir, tmp_dir,
- java64_home, retry = 5, interval = 10, solrconfig_content = None, jaas_file=None):
+ java64_home, retry = 5, interval = 10, solrconfig_content = None, jaas_file=None, java_opts=None):
"""
Upload configuration set to zookeeper with solrCloudCli.sh
At first, it tries to download configuration set if exists into a temporary location, then upload that one to
@@ -57,7 +59,7 @@ def upload_configuration_to_zk(zookeeper_quorum, solr_znode, config_set, config_
"""
random_num = random.random()
tmp_config_set_dir = format('{tmp_dir}/solr_config_{config_set}_{random_num}')
- solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home)
+ solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts)
Execute(format('{solr_cli_prefix} --download-config --config-dir {tmp_config_set_dir} --config-set {config_set} --retry {retry} --interval {interval}'),
only_if=format("{solr_cli_prefix} --check-config --config-set {config_set} --retry {retry} --interval {interval}"))
appendableDict = {}
@@ -88,7 +90,7 @@ def create_collection(zookeeper_quorum, solr_znode, collection, config_set, java
shards = 1, replication_factor = 1, max_shards = 1, retry = 5, interval = 10,
router_name = None, router_field = None, jaas_file = None, key_store_location = None,
key_store_password = None, key_store_type = None, trust_store_location = None,
- trust_store_password = None, trust_store_type = None):
+ trust_store_password = None, trust_store_type = None, java_opts=None):
"""
Create Solr collection based on a configuration set in zookeeper.
If this method called again the with higher shard number (or max_shard number), then it will indicate
@@ -98,7 +100,7 @@ def create_collection(zookeeper_quorum, solr_znode, collection, config_set, java
If you would like to add shards later to a collection, then use implicit routing, e.g.:
router_name = "implicit", router_field = "_router_field_"
"""
- solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home)
+ solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts)
if max_shards == 1: # if max shards is not specified use this strategy
max_shards = replication_factor * shards
@@ -121,59 +123,59 @@ def create_collection(zookeeper_quorum, solr_znode, collection, config_set, java
Execute(create_collection_cmd)
-def setup_kerberos(zookeeper_quorum, solr_znode, copy_from_znode, java64_home, secure=False, jaas_file=None):
+def setup_kerberos(zookeeper_quorum, solr_znode, copy_from_znode, java64_home, secure=False, jaas_file=None, java_opts=None):
"""
Copy all unsecured (or secured) Znode content to a secured (or unsecured) Znode,
and restrict the world permissions there.
"""
- solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
+ solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts, True)
setup_kerberos_cmd = format('{solr_cli_prefix} --setup-kerberos --copy-from-znode {copy_from_znode}')
if secure and jaas_file is not None:
setup_kerberos_cmd+=format(' --secure --jaas-file {jaas_file}')
Execute(setup_kerberos_cmd)
-def check_znode(zookeeper_quorum, solr_znode, java64_home, retry = 5, interval = 10):
+def check_znode(zookeeper_quorum, solr_znode, java64_home, retry = 5, interval = 10, java_opts=None):
"""
Check znode exists or not, throws exception if does not accessible.
"""
- solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
+ solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts, True)
check_znode_cmd = format('{solr_cli_prefix} --check-znode --retry {retry} --interval {interval}')
Execute(check_znode_cmd)
-def create_znode(zookeeper_quorum, solr_znode, java64_home, retry = 5 , interval = 10):
+def create_znode(zookeeper_quorum, solr_znode, java64_home, retry = 5 , interval = 10, java_opts=None):
"""
Create znode if does not exists, throws exception if zookeeper is not accessible.
"""
- solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
+ solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts, True)
create_znode_cmd = format('{solr_cli_prefix} --create-znode --retry {retry} --interval {interval}')
Execute(create_znode_cmd)
-def setup_kerberos_plugin(zookeeper_quorum, solr_znode, java64_home, secure=False, security_json_location = None, jaas_file = None):
+def setup_kerberos_plugin(zookeeper_quorum, solr_znode, java64_home, secure=False, security_json_location = None, jaas_file = None, java_opts=None):
"""
Set Kerberos plugin on the Solr znode in security.json, if secure is False, then clear the security.json
"""
- solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
+ solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts, True)
setup_kerberos_plugin_cmd = format('{solr_cli_prefix} --setup-kerberos-plugin')
if secure and jaas_file is not None and security_json_location is not None:
setup_kerberos_plugin_cmd+=format(' --jaas-file {jaas_file} --secure --security-json-location {security_json_location}')
Execute(setup_kerberos_plugin_cmd)
-def set_cluster_prop(zookeeper_quorum, solr_znode, prop_name, prop_value, java64_home, jaas_file = None):
+def set_cluster_prop(zookeeper_quorum, solr_znode, prop_name, prop_value, java64_home, jaas_file = None, java_opts=None):
"""
Set a cluster property on the Solr znode in clusterprops.json
"""
- solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home)
+ solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts)
set_cluster_prop_cmd = format('{solr_cli_prefix} --cluster-prop --property-name {prop_name} --property-value {prop_value}')
if jaas_file is not None:
set_cluster_prop_cmd+=format(' --jaas-file {jaas_file}')
Execute(set_cluster_prop_cmd)
-def secure_znode(config, zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_users=[], retry = 5 , interval = 10):
+def secure_znode(config, zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_users=[], retry = 5 , interval = 10, java_opts=None):
"""
Secure znode, set a list of sasl users acl to 'cdrwa', and set acl to 'r' only for the world.
Add infra-solr user by default if its available.
"""
- solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
+ solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts, True)
if "infra-solr-env" in config['configurations']:
sasl_users.append(__get_name_from_principal(config['configurations']['infra-solr-env']['infra_solr_kerberos_principal']))
sasl_users_str = ",".join(str(__get_name_from_principal(x)) for x in sasl_users)
@@ -181,20 +183,20 @@ def secure_znode(config, zookeeper_quorum, solr_znode, jaas_file, java64_home, s
Execute(secure_znode_cmd)
-def secure_solr_znode(zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_users_str=''):
+def secure_solr_znode(zookeeper_quorum, solr_znode, jaas_file, java64_home, sasl_users_str='', java_opts=None):
"""
Secure solr znode - setup acls to 'cdrwa' for solr user, set 'r' only for the world, skipping /znode/configs and znode/collections (set those to 'cr' for the world)
sasl_users_str: comma separated sasl users
"""
- solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, True)
+ solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts, True)
secure_solr_znode_cmd = format('{solr_cli_prefix} --secure-solr-znode --jaas-file {jaas_file} --sasl-users {sasl_users_str}')
Execute(secure_solr_znode_cmd)
-def remove_admin_handlers(zookeeper_quorum, solr_znode, java64_home, collection, jaas_file, retry = 5, interval = 10):
+def remove_admin_handlers(zookeeper_quorum, solr_znode, java64_home, collection, jaas_file, retry = 5, interval = 10, java_opts=None):
"""
Remove "solr.admin.AdminHandlers" request handler from collection config. Required for migrating to Solr 6 from Solr 5.
"""
- solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home)
+ solr_cli_prefix = __create_solr_cloud_cli_prefix(zookeeper_quorum, solr_znode, java64_home, java_opts)
remove_admin_handlers_cmd = format('{solr_cli_prefix} --remove-admin-handlers --collection {collection} --retry {retry} --interval {interval}')
if jaas_file is not None:
remove_admin_handlers_cmd+=format(' --jaas-file {jaas_file}')
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
index 7976587..372a894 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/tar_archive.py
@@ -25,7 +25,7 @@ from contextlib import closing
from resource_management.core.resources.system import Execute
def archive_dir(output_filename, input_dir):
- Execute(('tar', '-zcvf', output_filename, input_dir),
+ Execute(('tar', '-zcf', output_filename, '-C', input_dir, '.'),
sudo = True,
tries = 3,
try_sleep = 1,
@@ -40,7 +40,7 @@ def archive_directory_dereference(archive, directory):
:return: None
"""
- Execute(('tar', '-zcvhf', archive, directory),
+ Execute(('tar', '-zchf', archive, '-C', directory, '.'),
sudo = True,
tries = 3,
try_sleep = 1,
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 88e5336..1faaf51 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -53,7 +53,7 @@ from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions import stack_tools
from resource_management.libraries.functions.constants import Direction
from resource_management.libraries.script.config_dictionary import ConfigDictionary, UnknownConfiguration
-from resource_management.libraries.functions.repository_util import CommandRepository
+from resource_management.libraries.functions.repository_util import CommandRepository, RepositoryUtil
from resource_management.core.resources.system import Execute
from contextlib import closing
from resource_management.libraries.functions.stack_features import check_stack_feature
@@ -356,6 +356,14 @@ class Script(object):
Logger.logger.exception("Can not read json file with command parameters: ")
sys.exit(1)
+ from resource_management.libraries.functions import lzo_utils
+
+ repo_tags_to_skip = set()
+ if not lzo_utils.is_gpl_license_accepted():
+ repo_tags_to_skip.add("GPL")
+
+ Script.repository_util = RepositoryUtil(Script.config, repo_tags_to_skip)
+
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/HttpResponse.java
similarity index 72%
copy from ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
copy to ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/HttpResponse.java
index e7b9e77..3d8711b 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
+++ b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/HttpResponse.java
@@ -16,10 +16,22 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.infra.job;
+package org.apache.ambari.infra;
-import java.util.Map;
+public class HttpResponse {
+ private final int code;
+ private final String body;
-public interface PropertyMap<T extends JobProperties<T>> {
- Map<String, T> getPropertyMap();
+ public HttpResponse(int code, String body) {
+ this.code = code;
+ this.body = body;
+ }
+
+ public int getCode() {
+ return code;
+ }
+
+ public String getBody() {
+ return body;
+ }
}
diff --git a/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraClient.java b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraClient.java
index b798ce1..0118c76 100644
--- a/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraClient.java
+++ b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/InfraClient.java
@@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.IOUtils;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
@@ -71,11 +72,14 @@ public class InfraClient implements AutoCloseable {
execute(new HttpGet(baseUrl));
}
- private String execute(HttpRequestBase post) {
+ private HttpResponse execute(HttpRequestBase post) {
try (CloseableHttpResponse response = httpClient.execute(post)) {
String responseBodyText = IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset());
- LOG.info("Response code {} body {} ", response.getStatusLine().getStatusCode(), responseBodyText);
- return responseBodyText;
+ int statusCode = response.getStatusLine().getStatusCode();
+ LOG.info("Response code {} body {} ", statusCode, responseBodyText);
+ if (!(200 <= statusCode && statusCode <= 299))
+ throw new RuntimeException("Error while executing http request: " + responseBodyText);
+ return new HttpResponse(statusCode, responseBodyText);
} catch (ClientProtocolException e) {
throw new RuntimeException(e);
} catch (IOException e) {
@@ -83,16 +87,16 @@ public class InfraClient implements AutoCloseable {
}
}
- public String startJob(String jobName, String parameters) {
+ public JobExecutionInfo startJob(String jobName, String parameters) {
URIBuilder uriBuilder = new URIBuilder(baseUrl);
uriBuilder.setScheme("http");
uriBuilder.setPath(uriBuilder.getPath() + "/" + jobName);
if (!isBlank(parameters))
uriBuilder.addParameter("params", parameters);
try {
- String responseText = execute(new HttpPost(uriBuilder.build()));
+ String responseText = execute(new HttpPost(uriBuilder.build())).getBody();
Map<String, Object> responseContent = new ObjectMapper().readValue(responseText, new TypeReference<HashMap<String,Object>>() {});
- return responseContent.get("jobId").toString();
+ return new JobExecutionInfo(responseContent.get("jobId").toString(), ((Map)responseContent.get("jobExecutionData")).get("id").toString());
} catch (URISyntaxException | JsonParseException | JsonMappingException e) {
throw new RuntimeException(e);
} catch (IOException e) {
@@ -106,7 +110,21 @@ public class InfraClient implements AutoCloseable {
uriBuilder.setPath(String.format("%s/%s/%s/executions", uriBuilder.getPath(), jobName, jobId));
uriBuilder.addParameter("operation", "RESTART");
try {
- execute(new HttpPost(uriBuilder.build()));
+ HttpResponse httpResponse = execute(new HttpPost(uriBuilder.build()));
+ if (httpResponse.getCode() != 200)
+ throw new RuntimeException(httpResponse.getBody());
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void stopJob(String jobExecutionId) {
+ URIBuilder uriBuilder = new URIBuilder(baseUrl);
+ uriBuilder.setScheme("http");
+ uriBuilder.setPath(String.format("%s/executions/%s", uriBuilder.getPath(), jobExecutionId));
+ uriBuilder.addParameter("operation", "STOP");
+ try {
+ execute(new HttpDelete(uriBuilder.build()));
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditLogRequest.java b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/JobExecutionInfo.java
similarity index 60%
copy from ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditLogRequest.java
copy to ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/JobExecutionInfo.java
index 2abc69f..92b7834 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditLogRequest.java
+++ b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/JobExecutionInfo.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -16,25 +16,30 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.logsearch.model.request.impl;
+package org.apache.ambari.infra;
-import org.apache.ambari.logsearch.common.LogSearchConstants;
-import org.apache.ambari.logsearch.model.request.FieldParamDefinition;
+public class JobExecutionInfo {
+ private final String jobId;
+ private final String executionId;
-import javax.ws.rs.QueryParam;
-
-public class FieldAuditLogRequest extends BaseLogRequest implements FieldParamDefinition {
+ public JobExecutionInfo(String jobId, String executionId) {
+ this.jobId = jobId;
+ this.executionId = executionId;
+ }
- @QueryParam(LogSearchConstants.REQUEST_PARAM_FIELD)
- private String field;
+ public String getJobId() {
+ return jobId;
+ }
- @Override
- public String getField() {
- return field;
+ public String getExecutionId() {
+ return executionId;
}
@Override
- public void setField(String field) {
- this.field = field;
+ public String toString() {
+ return "JobExecutionInfo{" +
+ "jobId='" + jobId + '\'' +
+ ", executionId='" + executionId + '\'' +
+ '}';
}
}
diff --git a/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/AbstractInfraSteps.java b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/AbstractInfraSteps.java
index ece1c59..fb8a100 100644
--- a/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/AbstractInfraSteps.java
+++ b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/AbstractInfraSteps.java
@@ -64,6 +64,7 @@ public abstract class AbstractInfraSteps {
private static final int FAKE_S3_PORT = 4569;
private static final int HDFS_PORT = 9000;
private static final String AUDIT_LOGS_COLLECTION = "audit_logs";
+ private static final String HADOOP_LOGS_COLLECTION = "hadoop_logs";
protected static final String S3_BUCKET_NAME = "testbucket";
private String ambariFolder;
private String shellScriptLocation;
@@ -111,8 +112,8 @@ public abstract class AbstractInfraSteps {
SOLR_PORT,
AUDIT_LOGS_COLLECTION)).build();
- LOG.info("Creating collection");
- runCommand(new String[]{"docker", "exec", "docker_solr_1", "solr", "create_collection", "-c", AUDIT_LOGS_COLLECTION, "-d", "configsets/"+ AUDIT_LOGS_COLLECTION +"/conf", "-n", AUDIT_LOGS_COLLECTION + "_conf"});
+ createSolrCollection(AUDIT_LOGS_COLLECTION);
+ createSolrCollection(HADOOP_LOGS_COLLECTION);
LOG.info("Initializing s3 client");
s3client = new AmazonS3Client(new BasicAWSCredentials("remote-identity", "remote-credential"));
@@ -122,6 +123,11 @@ public abstract class AbstractInfraSteps {
checkInfraManagerReachable();
}
+ private void createSolrCollection(String collectionName) {
+ LOG.info("Creating collection");
+ runCommand(new String[]{"docker", "exec", "docker_solr_1", "solr", "create_collection", "-c", collectionName, "-d", "configsets/"+ collectionName +"/conf", "-n", collectionName + "_conf"});
+ }
+
private void runCommand(String[] command) {
try {
LOG.info("Exec command: {}", StringUtils.join(command, " "));
@@ -146,7 +152,7 @@ public abstract class AbstractInfraSteps {
});
}
- private void doWithin(int sec, String actionName, Runnable runnable) {
+ protected void doWithin(int sec, String actionName, Runnable runnable) {
long start = currentTimeMillis();
Exception exception;
while (true) {
diff --git a/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/ExportJobsSteps.java b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/ExportJobsSteps.java
index 7e54a31..d4224c6 100644
--- a/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/ExportJobsSteps.java
+++ b/ambari-infra/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/steps/ExportJobsSteps.java
@@ -23,6 +23,7 @@ import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import org.apache.ambari.infra.InfraClient;
+import org.apache.ambari.infra.JobExecutionInfo;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
@@ -56,7 +57,7 @@ import static org.junit.Assert.assertThat;
public class ExportJobsSteps extends AbstractInfraSteps {
private static final Logger LOG = LoggerFactory.getLogger(ExportJobsSteps.class);
- private Map<String, String> launchedJobs = new HashMap<>();
+ private Map<String, JobExecutionInfo> launchedJobs = new HashMap<>();
@Given("$count documents in solr")
public void addDocuments(int count) throws Exception {
@@ -85,22 +86,39 @@ public class ExportJobsSteps extends AbstractInfraSteps {
@When("start $jobName job")
public void startJob(String jobName) throws Exception {
- startJob(jobName, null);
+ startJob(jobName, null, 0);
}
- @When("start $jobName job with parameters $parameters")
- public void startJob(String jobName, String parameters) throws Exception {
+ @When("start $jobName job with parameters $parameters after $waitSec seconds")
+ public void startJob(String jobName, String parameters, int waitSec) throws Exception {
+ Thread.sleep(waitSec * 1000);
try (InfraClient httpClient = getInfraClient()) {
- String jobId = httpClient.startJob(jobName, parameters);
- LOG.info("Job {} started jobId: {}", jobName, jobId);
- launchedJobs.put(jobName, jobId);
+ JobExecutionInfo jobExecutionInfo = httpClient.startJob(jobName, parameters);
+ LOG.info("Job {} started: {}", jobName, jobExecutionInfo);
+ launchedJobs.put(jobName, jobExecutionInfo);
}
}
- @When("restart $jobName job")
- public void restartJob(String jobName) throws Exception {
+ @When("restart $jobName job within $waitSec seconds")
+ public void restartJob(String jobName, int waitSec) {
+ doWithin(waitSec, "Restarting job " + jobName, () -> {
+ try (InfraClient httpClient = getInfraClient()) {
+ httpClient.restartJob(jobName, launchedJobs.get(jobName).getJobId());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @When("stop job $jobName after at least $count file exists in s3 with filename containing text $text within $waitSec seconds")
+ public void stopJob(String jobName, int count, String text, int waitSec) throws Exception {
+ AmazonS3Client s3Client = getS3client();
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(S3_BUCKET_NAME);
+ doWithin(waitSec, "check uploaded files to s3", () -> s3Client.doesBucketExist(S3_BUCKET_NAME)
+ && fileCountOnS3(text, s3Client, listObjectsRequest) > count);
+
try (InfraClient httpClient = getInfraClient()) {
- httpClient.restartJob(jobName, launchedJobs.get(jobName));
+ httpClient.stopJob(launchedJobs.get(jobName).getExecutionId());
}
}
@@ -125,9 +143,25 @@ public class ExportJobsSteps extends AbstractInfraSteps {
AmazonS3Client s3Client = getS3client();
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(S3_BUCKET_NAME);
doWithin(waitSec, "check uploaded files to s3", () -> s3Client.doesBucketExist(S3_BUCKET_NAME)
- && s3Client.listObjects(listObjectsRequest).getObjectSummaries().stream()
- .filter(s3ObjectSummary -> s3ObjectSummary.getKey().contains(text))
- .count() == count);
+ && fileCountOnS3(text, s3Client, listObjectsRequest) == count);
+ }
+
+ private long fileCountOnS3(String text, AmazonS3Client s3Client, ListObjectsRequest listObjectsRequest) {
+ return s3Client.listObjects(listObjectsRequest).getObjectSummaries().stream()
+ .filter(s3ObjectSummary -> s3ObjectSummary.getKey().contains(text))
+ .count();
+ }
+
+ @Then("Less than $count files exists on s3 server with filenames containing the text $text after $waitSec seconds")
+ public void checkLessThanFileExistsOnS3(long count, String text, int waitSec) {
+ AmazonS3Client s3Client = getS3client();
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(S3_BUCKET_NAME);
+ doWithin(waitSec, "check uploaded files to s3", () -> s3Client.doesBucketExist(S3_BUCKET_NAME) && between(
+ fileCountOnS3(text, s3Client, listObjectsRequest), 1L, count - 1L));
+ }
+
+ private boolean between(long count, long from, long to) {
+ return from <= count && count <= to;
}
@Then("No file exists on s3 server with filenames containing the text $text")
@@ -184,9 +218,9 @@ public class ExportJobsSteps extends AbstractInfraSteps {
}
}
- @Then("Check $count files exists on local filesystem with filenames containing the text $text in the folder $path")
- public void checkNumberOfFilesOnLocalFilesystem(long count, String text, String path) {
- File destinationDirectory = new File(getLocalDataFolder(), path);
+ @Then("Check $count files exists on local filesystem with filenames containing the text $text in the folder $path for job $jobName")
+ public void checkNumberOfFilesOnLocalFilesystem(long count, String text, String path, String jobName) {
+ File destinationDirectory = new File(getLocalDataFolder(), path.replace("${jobId}", launchedJobs.get(jobName).getJobId()));
LOG.info("Destination directory path: {}", destinationDirectory.getAbsolutePath());
doWithin(5, "Destination directory exists", destinationDirectory::exists);
diff --git a/ambari-infra/ambari-infra-manager-it/src/test/resources/stories/infra_api_tests.story b/ambari-infra/ambari-infra-manager-it/src/test/resources/stories/infra_api_tests.story
index d1eb4a4..122a634 100644
--- a/ambari-infra/ambari-infra-manager-it/src/test/resources/stories/infra_api_tests.story
+++ b/ambari-infra/ambari-infra-manager-it/src/test/resources/stories/infra_api_tests.story
@@ -8,7 +8,7 @@ Then Check filenames contains the text audit_logs on s3 server after 20 seconds
Scenario: Exporting 10 documents using writeBlockSize=3 produces 4 files
Given 10 documents in solr with logtime from 2010-10-09T05:00:00.000Z to 2010-10-09T20:00:00.000Z
-When start archive_audit_logs job with parameters writeBlockSize=3,start=2010-10-09T00:00:00.000Z,end=2010-10-11T00:00:00.000Z
+When start archive_audit_logs job with parameters writeBlockSize=3,start=2010-10-09T00:00:00.000Z,end=2010-10-11T00:00:00.000Z after 2 seconds
Then Check 4 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2010-10-09 after 20 seconds
And solr does not contain documents between 2010-10-09T05:00:00.000Z and 2010-10-09T20:00:00.000Z after 5 seconds
@@ -16,7 +16,7 @@ And solr does not contain documents between 2010-10-09T05:00:00.000Z and 2010-10
Scenario: Running archiving job with a bigger start value than end value exports and deletes 0 documents
Given 10 documents in solr with logtime from 2010-01-01T05:00:00.000Z to 2010-01-04T05:00:00.000Z
-When start archive_audit_logs job with parameters writeBlockSize=3,start=2010-01-03T05:00:00.000Z,end=2010-01-02T05:00:00.000Z
+When start archive_audit_logs job with parameters writeBlockSize=3,start=2010-01-03T05:00:00.000Z,end=2010-01-02T05:00:00.000Z after 2 seconds
Then No file exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2010-01-0
And solr contains 10 documents between 2010-01-01T05:00:00.000Z and 2010-01-04T05:00:00.000Z
@@ -25,11 +25,11 @@ Scenario: Archiving job fails when part of the data is exported. After resolving
Given 200 documents in solr with logtime from 2011-10-09T05:00:00.000Z to 2011-10-09T20:00:00.000Z
And a file on s3 with key solr_archive_audit_logs_-_2011-10-09T08-00-00.000Z.json.tar.gz
-When start archive_audit_logs job with parameters writeBlockSize=20,start=2010-11-09T00:00:00.000Z,end=2011-10-11T00:00:00.000Z
+When start archive_audit_logs job with parameters writeBlockSize=20,start=2010-11-09T00:00:00.000Z,end=2011-10-11T00:00:00.000Z after 2 seconds
Then Check 3 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2011-10-09 after 20 seconds
And solr does not contain documents between 2011-10-09T05:00:00.000Z and 2011-10-09T07:59:59.999Z after 5 seconds
When delete file with key solr_archive_audit_logs_-_2011-10-09T08-00-00.000Z.json.tar.gz from s3
-And restart archive_audit_logs job
+And restart archive_audit_logs job within 2 seconds
Then Check 10 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2011-10-09 after 20 seconds
And solr does not contain documents between 2011-10-09T05:00:00.000Z and 2011-10-09T20:00:00.000Z after 5 seconds
@@ -37,14 +37,14 @@ And solr does not contain documents between 2011-10-09T05:00:00.000Z and 2011-10
Scenario: After Deleting job deletes documents from solr no document found in the specified interval
Given 10 documents in solr with logtime from 2012-10-09T05:00:00.000Z to 2012-10-09T20:00:00.000Z
-When start delete_audit_logs job with parameters start=2012-10-09T05:00:00.000Z,end=2012-10-09T20:00:00.000Z
+When start delete_audit_logs job with parameters start=2012-10-09T05:00:00.000Z,end=2012-10-09T20:00:00.000Z after 2 seconds
Then solr does not contain documents between 2012-10-09T05:00:00.000Z and 2012-10-09T20:00:00.000Z after 5 seconds
Scenario: Archiving documents to hdfs
Given 1000 documents in solr with logtime from 2014-01-04T05:00:00.000Z to 2014-01-06T20:00:00.000Z
-When start archive_audit_logs job with parameters start=2014-01-04T05:00:00.000Z,end=2014-01-06T20:00:00.000Z,destination=HDFS
+When start archive_audit_logs job with parameters start=2014-01-04T05:00:00.000Z,end=2014-01-06T20:00:00.000Z,destination=HDFS after 2 seconds
Then Check 7 files exists on hdfs with filenames containing the text audit_logs_-_2014-01-0 in the folder /test_audit_logs after 10 seconds
And solr does not contain documents between 2014-01-04T05:00:00.000Z and 2014-01-06T20:00:00.000Z after 10 seconds
@@ -52,6 +52,16 @@ And solr does not contain documents between 2014-01-04T05:00:00.000Z and 2014-01
Scenario: Archiving documents to local filesystem
Given 200 documents in solr with logtime from 2014-02-04T05:00:00.000Z to 2014-02-06T20:00:00.000Z
-When start archive_audit_logs job with parameters start=2014-02-04T05:00:00.000Z,end=2014-02-06T20:00:00.000Z,destination=LOCAL,localDestinationDirectory=/root/archive
-Then Check 2 files exists on local filesystem with filenames containing the text audit_logs_-_2014-02-0 in the folder audit_logs_8_2014-02-06T20-00-00.000Z
+When start archive_audit_logs job with parameters start=2014-02-04T05:00:00.000Z,end=2014-02-06T20:00:00.000Z,destination=LOCAL,localDestinationDirectory=/root/archive after 2 seconds
+Then Check 2 files exists on local filesystem with filenames containing the text audit_logs_-_2014-02-0 in the folder audit_logs_${jobId}_2014-02-06T20-00-00.000Z for job archive_audit_logs
And solr does not contain documents between 2014-02-04T05:00:00.000Z and 2014-02-06T20:00:00.000Z after 10 seconds
+
+
+Scenario: Launch Archiving job. Initiate stop and check that part of the data is archived. After restart all data must be extracted.
+
+Given 200 documents in solr with logtime from 2014-03-09T05:00:00.000Z to 2014-03-09T20:00:00.000Z
+When start archive_audit_logs job with parameters writeBlockSize=20,start=2014-03-09T05:00:00.000Z,end=2014-03-09T20:00:00.000Z after 2 seconds
+And stop job archive_audit_logs after at least 1 file exists in s3 with filename containing text solr_archive_audit_logs_-_2014-03-09 within 10 seconds
+Then Less than 10 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2014-03-09 after 20 seconds
+When restart archive_audit_logs job within 10 seconds
+Then Check 10 files exists on s3 server with filenames containing the text solr_archive_audit_logs_-_2014-03-09 after 20 seconds
diff --git a/ambari-infra/ambari-infra-manager/docs/api/swagger.yaml b/ambari-infra/ambari-infra-manager/docs/api/swagger.yaml
index 824629f..6fad22d 100644
--- a/ambari-infra/ambari-infra-manager/docs/api/swagger.yaml
+++ b/ambari-infra/ambari-infra-manager/docs/api/swagger.yaml
@@ -65,7 +65,7 @@ paths:
- "jobs"
summary: "Get job and step details for job execution instance."
description: ""
- operationId: "getExectionInfo"
+ operationId: "getExecutionInfo"
produces:
- "application/json"
parameters:
diff --git a/ambari-infra/ambari-infra-manager/pom.xml b/ambari-infra/ambari-infra-manager/pom.xml
index 467dbad..5af84e6 100644
--- a/ambari-infra/ambari-infra-manager/pom.xml
+++ b/ambari-infra/ambari-infra-manager/pom.xml
@@ -338,7 +338,6 @@
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
- <version>1.3.1</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerSchedulingConfig.java
similarity index 65%
copy from ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
copy to ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerSchedulingConfig.java
index e7b9e77..bb495a2 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerSchedulingConfig.java
@@ -16,10 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.infra.job;
+package org.apache.ambari.infra.conf;
-import java.util.Map;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
-public interface PropertyMap<T extends JobProperties<T>> {
- Map<String, T> getPropertyMap();
+@Configuration
+public class InfraManagerSchedulingConfig {
+ @Bean
+ public TaskScheduler taskScheduler() {
+ return new ThreadPoolTaskScheduler();
+ }
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/batch/InfraManagerBatchConfig.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/batch/InfraManagerBatchConfig.java
index b1169b4..706ed8b 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/batch/InfraManagerBatchConfig.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/batch/InfraManagerBatchConfig.java
@@ -30,6 +30,7 @@ import org.springframework.batch.core.configuration.JobRegistry;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.support.JobRegistryBeanPostProcessor;
import org.springframework.batch.core.explore.JobExplorer;
+import org.springframework.batch.core.explore.support.JobExplorerFactoryBean;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.launch.JobOperator;
import org.springframework.batch.core.launch.support.SimpleJobLauncher;
@@ -56,7 +57,6 @@ import org.springframework.transaction.PlatformTransactionManager;
import javax.inject.Inject;
import javax.sql.DataSource;
-import java.net.MalformedURLException;
@Configuration
@EnableBatchProcessing
@@ -85,9 +85,6 @@ public class InfraManagerBatchConfig {
@Inject
private JobRegistry jobRegistry;
- @Inject
- private JobExplorer jobExplorer;
-
@Bean
public DataSource dataSource() {
DriverManagerDataSource dataSource = new DriverManagerDataSource();
@@ -99,8 +96,7 @@ public class InfraManagerBatchConfig {
}
@Bean
- public DataSourceInitializer dataSourceInitializer(DataSource dataSource)
- throws MalformedURLException {
+ public DataSourceInitializer dataSourceInitializer() {
ResourceDatabasePopulator databasePopulator = new ResourceDatabasePopulator();
if (dropDatabaseOnStartup) {
databasePopulator.addScript(dropRepositoryTables);
@@ -110,7 +106,7 @@ public class InfraManagerBatchConfig {
databasePopulator.setContinueOnError(true);
DataSourceInitializer initializer = new DataSourceInitializer();
- initializer.setDataSource(dataSource);
+ initializer.setDataSource(dataSource());
initializer.setDatabasePopulator(databasePopulator);
return initializer;
@@ -125,14 +121,14 @@ public class InfraManagerBatchConfig {
public JobRepository jobRepository() throws Exception {
JobRepositoryFactoryBean factory = new JobRepositoryFactoryBean();
factory.setDataSource(dataSource());
- factory.setTransactionManager(getTransactionManager());
+ factory.setTransactionManager(transactionManager());
factory.setSerializer(executionContextSerializer());
factory.afterPropertiesSet();
return factory.getObject();
}
@Bean
- public PlatformTransactionManager getTransactionManager() {
+ public PlatformTransactionManager transactionManager() {
return new ResourcelessTransactionManager();
}
@@ -148,7 +144,7 @@ public class InfraManagerBatchConfig {
@Bean
public JobOperator jobOperator() throws Exception {
SimpleJobOperator jobOperator = new SimpleJobOperator();
- jobOperator.setJobExplorer(jobExplorer);
+ jobOperator.setJobExplorer(jobExplorer());
jobOperator.setJobLauncher(jobLauncher());
jobOperator.setJobRegistry(jobRegistry);
jobOperator.setJobRepository(jobRepository());
@@ -156,6 +152,15 @@ public class InfraManagerBatchConfig {
}
@Bean
+ public JobExplorer jobExplorer() throws Exception {
+ JobExplorerFactoryBean factoryBean = new JobExplorerFactoryBean();
+ factoryBean.setSerializer(executionContextSerializer());
+ factoryBean.setDataSource(dataSource());
+ factoryBean.afterPropertiesSet();
+ return factoryBean.getObject();
+ }
+
+ @Bean
public JobRegistryBeanPostProcessor jobRegistryBeanPostProcessor() {
JobRegistryBeanPostProcessor jobRegistryBeanPostProcessor = new JobRegistryBeanPostProcessor();
jobRegistryBeanPostProcessor.setJobRegistry(jobRegistry);
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditLogRequest.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java
similarity index 57%
copy from ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditLogRequest.java
copy to ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java
index 2abc69f..6d32963 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditLogRequest.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -16,25 +16,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.logsearch.model.request.impl;
+package org.apache.ambari.infra.conf.security;
-import org.apache.ambari.logsearch.common.LogSearchConstants;
-import org.apache.ambari.logsearch.model.request.FieldParamDefinition;
+import java.util.Optional;
-import javax.ws.rs.QueryParam;
+public class CompositePasswordStore implements PasswordStore {
+ private PasswordStore[] passwordStores;
-public class FieldAuditLogRequest extends BaseLogRequest implements FieldParamDefinition {
-
- @QueryParam(LogSearchConstants.REQUEST_PARAM_FIELD)
- private String field;
-
- @Override
- public String getField() {
- return field;
+ public CompositePasswordStore(PasswordStore... passwordStores) {
+ this.passwordStores = passwordStores;
}
@Override
- public void setField(String field) {
- this.field = field;
+ public Optional<String> getPassword(String propertyName) {
+ for (PasswordStore passwordStore : passwordStores) {
+ Optional<String> optionalPassword = passwordStore.getPassword(propertyName);
+ if (optionalPassword.isPresent())
+ return optionalPassword;
+ }
+ return Optional.empty();
}
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java
new file mode 100644
index 0000000..9e1a17f
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.conf.security;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Optional;
+
+import static org.apache.commons.lang.StringUtils.isBlank;
+import static org.apache.commons.lang3.ArrayUtils.isNotEmpty;
+
+public class HadoopCredentialStore implements PasswordStore {
+ private static final Logger LOG = LoggerFactory.getLogger(InfraManagerSecurityConfig.class);
+ public static final String CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY = "hadoop.security.credential.provider.path";
+
+ private final String credentialStoreProviderPath;
+
+ public HadoopCredentialStore(String credentialStoreProviderPath) {
+ this.credentialStoreProviderPath = credentialStoreProviderPath;
+ }
+
+ @Override
+ public Optional<String> getPassword(String propertyName) {
+ try {
+ if (isBlank(credentialStoreProviderPath)) {
+ return Optional.empty();
+ }
+
+ org.apache.hadoop.conf.Configuration config = new org.apache.hadoop.conf.Configuration();
+ config.set(CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY, credentialStoreProviderPath);
+ char[] passwordChars = config.getPassword(propertyName);
+ return (isNotEmpty(passwordChars)) ? Optional.of(new String(passwordChars)) : Optional.empty();
+ } catch (Exception e) {
+ LOG.warn("Could not load password {} from credential store.", propertyName);
+ return Optional.empty();
+ }
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditBarGraphRequest.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java
similarity index 54%
copy from ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditBarGraphRequest.java
copy to ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java
index aa08aaf..45b79b3 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditBarGraphRequest.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -16,25 +16,23 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.logsearch.model.request.impl;
+package org.apache.ambari.infra.conf.security;
-import org.apache.ambari.logsearch.common.LogSearchConstants;
-import org.apache.ambari.logsearch.model.request.FieldParamDefinition;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
-import javax.ws.rs.QueryParam;
+import static org.apache.ambari.infra.conf.security.HadoopCredentialStore.CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY;
-public class FieldAuditBarGraphRequest extends AuditBarGraphRequest implements FieldParamDefinition {
+@Configuration
+public class InfraManagerSecurityConfig {
- @QueryParam(LogSearchConstants.REQUEST_PARAM_FIELD)
- private String field;
+ @Value("${"+ CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY + ":}")
+ private String credentialStoreProviderPath;
- @Override
- public String getField() {
- return field;
- }
- @Override
- public void setField(String field) {
- this.field = field;
+ @Bean
+ public PasswordStore passwords() {
+ return new CompositePasswordStore(new HadoopCredentialStore(credentialStoreProviderPath), new SecurityEnvironment());
}
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java
similarity index 83%
copy from ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
copy to ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java
index e7b9e77..19848fe 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java
@@ -16,10 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.infra.job;
+package org.apache.ambari.infra.conf.security;
-import java.util.Map;
+import java.util.Optional;
-public interface PropertyMap<T extends JobProperties<T>> {
- Map<String, T> getPropertyMap();
+public interface PasswordStore {
+ Optional<String> getPassword(String propertyName);
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
similarity index 74%
copy from ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
copy to ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
index e7b9e77..8e3387b 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
@@ -16,10 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.infra.job;
+package org.apache.ambari.infra.conf.security;
-import java.util.Map;
+import java.util.Optional;
-public interface PropertyMap<T extends JobProperties<T>> {
- Map<String, T> getPropertyMap();
+public class SecurityEnvironment implements PasswordStore {
+ @Override
+ public Optional<String> getPassword(String propertyName) {
+ return Optional.ofNullable(System.getenv(propertyName));
+ }
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/AbstractJobsConfiguration.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/AbstractJobsConfiguration.java
new file mode 100644
index 0000000..02a6885
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/AbstractJobsConfiguration.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
+import org.springframework.batch.core.configuration.support.JobRegistryBeanPostProcessor;
+import org.springframework.batch.core.job.builder.JobBuilder;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.event.EventListener;
+
+import javax.annotation.PostConstruct;
+import java.util.Map;
+
+public abstract class AbstractJobsConfiguration<T extends JobProperties<T>> {
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractJobsConfiguration.class);
+
+ private final Map<String, T> propertyMap;
+ private final JobScheduler scheduler;
+ private final JobBuilderFactory jobs;
+ private final JobRegistryBeanPostProcessor jobRegistryBeanPostProcessor;
+
+ protected AbstractJobsConfiguration(Map<String, T> propertyMap, JobScheduler scheduler, JobBuilderFactory jobs, JobRegistryBeanPostProcessor jobRegistryBeanPostProcessor) {
+ this.propertyMap = propertyMap;
+ this.scheduler = scheduler;
+ this.jobs = jobs;
+ this.jobRegistryBeanPostProcessor = jobRegistryBeanPostProcessor;
+ }
+
+ @PostConstruct
+ public void registerJobs() {
+ if (propertyMap == null)
+ return;
+
+ propertyMap.keySet().stream()
+ .filter(key -> propertyMap.get(key).isEnabled())
+ .forEach(jobName -> {
+ try {
+ propertyMap.get(jobName).validate(jobName);
+ LOG.info("Registering job {}", jobName);
+ JobBuilder jobBuilder = jobs.get(jobName).listener(new JobsPropertyMap<>(propertyMap));
+ Job job = buildJob(jobBuilder);
+ jobRegistryBeanPostProcessor.postProcessAfterInitialization(job, jobName);
+ }
+ catch (Exception e) {
+ LOG.warn("Unable to register job " + jobName, e);
+ propertyMap.get(jobName).setEnabled(false);
+ }
+ });
+ }
+
+ @EventListener(ApplicationReadyEvent.class)
+ public void scheduleJobs() {
+ if (propertyMap == null)
+ return;
+
+ propertyMap.keySet().stream()
+ .filter(key -> propertyMap.get(key).isEnabled())
+ .forEach(jobName -> propertyMap.get(jobName).scheduling().ifPresent(
+ schedulingProperties -> scheduler.schedule(jobName, schedulingProperties)));
+ }
+
+ protected abstract Job buildJob(JobBuilder jobBuilder);
+}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobConfigurationException.java
similarity index 84%
copy from ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
copy to ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobConfigurationException.java
index e7b9e77..8c16daa 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobConfigurationException.java
@@ -18,8 +18,8 @@
*/
package org.apache.ambari.infra.job;
-import java.util.Map;
-
-public interface PropertyMap<T extends JobProperties<T>> {
- Map<String, T> getPropertyMap();
+public class JobConfigurationException extends RuntimeException {
+ public JobConfigurationException(String message, Exception ex) {
+ super(message, ex);
+ }
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobContextRepository.java
similarity index 79%
copy from ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
copy to ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobContextRepository.java
index e7b9e77..eb7f717 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobContextRepository.java
@@ -18,8 +18,9 @@
*/
package org.apache.ambari.infra.job;
-import java.util.Map;
+import org.springframework.batch.core.StepExecution;
-public interface PropertyMap<T extends JobProperties<T>> {
- Map<String, T> getPropertyMap();
+public interface JobContextRepository {
+ StepExecution getStepExecution(Long jobExecutionId, Long id);
+ void updateExecutionContext(StepExecution stepExecution);
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobContextRepositoryImpl.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobContextRepositoryImpl.java
new file mode 100644
index 0000000..fbb256f
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobContextRepositoryImpl.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job;
+
+import org.springframework.batch.admin.service.JobService;
+import org.springframework.batch.admin.service.NoSuchStepExecutionException;
+import org.springframework.batch.core.StepExecution;
+import org.springframework.batch.core.launch.NoSuchJobExecutionException;
+import org.springframework.batch.core.repository.JobRepository;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+
+@Named
+public class JobContextRepositoryImpl implements JobContextRepository {
+
+ @Inject
+ private JobRepository jobRepository;
+ @Inject
+ private JobService jobService;
+
+
+ @Override
+ public StepExecution getStepExecution(Long jobExecutionId, Long id) {
+ try {
+ return jobService.getStepExecution(jobExecutionId, id);
+ } catch (NoSuchStepExecutionException | NoSuchJobExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void updateExecutionContext(StepExecution stepExecution) {
+ jobRepository.updateExecutionContext(stepExecution);
+ }
+}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobProperties.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobProperties.java
index 292e15e..79406d0 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobProperties.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobProperties.java
@@ -23,14 +23,32 @@ import org.springframework.batch.core.JobParameters;
import java.io.IOException;
import java.io.UncheckedIOException;
+import java.util.Optional;
public abstract class JobProperties<T extends JobProperties<T>> {
+
+ private SchedulingProperties scheduling;
private final Class<T> clazz;
+ private boolean enabled;
protected JobProperties(Class<T> clazz) {
this.clazz = clazz;
}
+ public SchedulingProperties getScheduling() {
+ return scheduling;
+ }
+
+ public Optional<SchedulingProperties> scheduling() {
+ if (scheduling != null && scheduling.isEnabled())
+ return Optional.of(scheduling);
+ return Optional.empty();
+ }
+
+ public void setScheduling(SchedulingProperties scheduling) {
+ this.scheduling = scheduling;
+ }
+
public T deepCopy() {
try {
ObjectMapper objectMapper = new ObjectMapper();
@@ -44,4 +62,21 @@ public abstract class JobProperties<T extends JobProperties<T>> {
public abstract void apply(JobParameters jobParameters);
public abstract void validate();
+
+ public void validate(String jobName) {
+ try {
+ validate();
+ }
+ catch (Exception ex) {
+ throw new JobConfigurationException(String.format("Configuration of job %s is invalid: %s!", jobName, ex.getMessage()), ex);
+ }
+ }
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobScheduler.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobScheduler.java
new file mode 100644
index 0000000..324c0b3
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobScheduler.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job;
+
+import org.apache.ambari.infra.manager.Jobs;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.batch.core.ExitStatus;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.core.JobParametersBuilder;
+import org.springframework.batch.core.JobParametersInvalidException;
+import org.springframework.batch.core.launch.NoSuchJobException;
+import org.springframework.batch.core.launch.NoSuchJobExecutionException;
+import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
+import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
+import org.springframework.batch.core.repository.JobRestartException;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.scheduling.support.CronTrigger;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import java.time.Duration;
+import java.time.OffsetDateTime;
+
+import static org.apache.ambari.infra.job.archive.FileNameSuffixFormatter.SOLR_DATETIME_FORMATTER;
+import static org.apache.commons.lang.StringUtils.isBlank;
+
+@Named
+public class JobScheduler {
+ private static final Logger LOG = LoggerFactory.getLogger(JobScheduler.class);
+
+ private final TaskScheduler scheduler;
+ private final Jobs jobs;
+
+ @Inject
+ public JobScheduler(TaskScheduler scheduler, Jobs jobs) {
+ this.scheduler = scheduler;
+ this.jobs = jobs;
+ }
+
+ public void schedule(String jobName, SchedulingProperties schedulingProperties) {
+ try {
+ jobs.lastRun(jobName).ifPresent(this::restartIfFailed);
+ } catch (NoSuchJobException | NoSuchJobExecutionException e) {
+ throw new RuntimeException(e);
+ }
+
+ scheduler.schedule(() -> launchJob(jobName, schedulingProperties.getIntervalEndDelta()), new CronTrigger(schedulingProperties.getCron()));
+ LOG.info("Job {} scheduled for running. Cron: {}", jobName, schedulingProperties.getCron());
+ }
+
+ private void restartIfFailed(JobExecution jobExecution) {
+ if (jobExecution.getExitStatus() == ExitStatus.FAILED) {
+ try {
+ jobs.restart(jobExecution.getId());
+ } catch (JobInstanceAlreadyCompleteException | NoSuchJobException | JobExecutionAlreadyRunningException | JobRestartException | JobParametersInvalidException | NoSuchJobExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private void launchJob(String jobName, String endDelta) {
+ try {
+ JobParametersBuilder jobParametersBuilder = new JobParametersBuilder();
+ if (!isBlank(endDelta))
+ jobParametersBuilder.addString("end", SOLR_DATETIME_FORMATTER.format(OffsetDateTime.now().minus(Duration.parse(endDelta))));
+
+ jobs.launchJob(jobName, jobParametersBuilder.toJobParameters());
+ } catch (JobParametersInvalidException | NoSuchJobException | JobExecutionAlreadyRunningException | JobRestartException | JobInstanceAlreadyCompleteException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobPropertyMap.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobsPropertyMap.java
similarity index 85%
rename from ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobPropertyMap.java
rename to ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobsPropertyMap.java
index b5061f8..094e797 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobPropertyMap.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobsPropertyMap.java
@@ -22,11 +22,13 @@ import org.springframework.batch.core.ExitStatus;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobExecutionListener;
-public class JobPropertyMap<T extends JobProperties<T>> implements JobExecutionListener {
+import java.util.Map;
- private final PropertyMap<T> propertyMap;
+public class JobsPropertyMap<T extends JobProperties<T>> implements JobExecutionListener {
- public JobPropertyMap(PropertyMap<T> propertyMap) {
+ private final Map<String, T> propertyMap;
+
+ public JobsPropertyMap(Map<String, T> propertyMap) {
this.propertyMap = propertyMap;
}
@@ -34,13 +36,13 @@ public class JobPropertyMap<T extends JobProperties<T>> implements JobExecutionL
public void beforeJob(JobExecution jobExecution) {
try {
String jobName = jobExecution.getJobInstance().getJobName();
- T defaultProperties = propertyMap.getPropertyMap().get(jobName);
+ T defaultProperties = propertyMap.get(jobName);
if (defaultProperties == null)
throw new UnsupportedOperationException("Properties not found for job " + jobName);
T properties = defaultProperties.deepCopy();
properties.apply(jobExecution.getJobParameters());
- properties.validate();
+ properties.validate(jobName);
jobExecution.getExecutionContext().put("jobProperties", properties);
}
catch (UnsupportedOperationException | IllegalArgumentException ex) {
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/SchedulingProperties.java
similarity index 60%
copy from ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
copy to ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/SchedulingProperties.java
index e7b9e77..af81b4f 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/SchedulingProperties.java
@@ -18,8 +18,32 @@
*/
package org.apache.ambari.infra.job;
-import java.util.Map;
+public class SchedulingProperties {
+ private boolean enabled = false;
+ private String cron;
+ private String intervalEndDelta;
-public interface PropertyMap<T extends JobProperties<T>> {
- Map<String, T> getPropertyMap();
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
+
+ public String getCron() {
+ return cron;
+ }
+
+ public void setCron(String cron) {
+ this.cron = cron;
+ }
+
+ public String getIntervalEndDelta() {
+ return intervalEndDelta;
+ }
+
+ public void setIntervalEndDelta(String intervalEndDelta) {
+ this.intervalEndDelta = intervalEndDelta;
+ }
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java
index 837b9c4..8358dd0 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java
@@ -19,7 +19,10 @@
package org.apache.ambari.infra.job.archive;
import org.apache.ambari.infra.conf.InfraManagerDataConfig;
-import org.apache.ambari.infra.job.JobPropertyMap;
+import org.apache.ambari.infra.conf.security.PasswordStore;
+import org.apache.ambari.infra.job.AbstractJobsConfiguration;
+import org.apache.ambari.infra.job.JobContextRepository;
+import org.apache.ambari.infra.job.JobScheduler;
import org.apache.ambari.infra.job.ObjectSource;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
@@ -31,55 +34,41 @@ import org.springframework.batch.core.configuration.annotation.JobScope;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.configuration.support.JobRegistryBeanPostProcessor;
+import org.springframework.batch.core.job.builder.JobBuilder;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
-import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.io.File;
import static org.apache.commons.lang.StringUtils.isBlank;
@Configuration
-public class DocumentArchivingConfiguration {
+public class DocumentArchivingConfiguration extends AbstractJobsConfiguration<DocumentArchivingProperties> {
private static final Logger LOG = LoggerFactory.getLogger(DocumentArchivingConfiguration.class);
private static final DocumentWiper NOT_DELETE = (firstDocument, lastDocument) -> { };
- @Inject
- private DocumentArchivingPropertyMap propertyMap;
-
- @Inject
- private StepBuilderFactory steps;
+ private final StepBuilderFactory steps;
+ private final Step exportStep;
@Inject
- private JobBuilderFactory jobs;
-
- @Inject
- @Qualifier("exportStep")
- private Step exportStep;
-
- @Inject
- private JobRegistryBeanPostProcessor jobRegistryBeanPostProcessor;
-
-
- @PostConstruct
- public void createJobs() {
- if (propertyMap == null || propertyMap.getSolrDataArchiving() == null)
- return;
-
- propertyMap.getSolrDataArchiving().values().forEach(DocumentArchivingProperties::validate);
-
- propertyMap.getSolrDataArchiving().keySet().forEach(jobName -> {
- LOG.info("Registering data archiving job {}", jobName);
- Job job = logExportJob(jobName, exportStep);
- jobRegistryBeanPostProcessor.postProcessAfterInitialization(job, jobName);
- });
+ public DocumentArchivingConfiguration(
+ DocumentArchivingPropertyMap jobsPropertyMap,
+ JobScheduler scheduler,
+ StepBuilderFactory steps,
+ JobBuilderFactory jobs,
+ @Qualifier("exportStep") Step exportStep,
+ JobRegistryBeanPostProcessor jobRegistryBeanPostProcessor) {
+ super(jobsPropertyMap.getSolrDataArchiving(), scheduler, jobs, jobRegistryBeanPostProcessor);
+ this.exportStep = exportStep;
+ this.steps = steps;
}
- private Job logExportJob(String jobName, Step logExportStep) {
- return jobs.get(jobName).listener(new JobPropertyMap<>(propertyMap)).start(logExportStep).build();
+ @Override
+ protected Job buildJob(JobBuilder jobBuilder) {
+ return jobBuilder.start(exportStep).build();
}
@Bean
@@ -93,17 +82,21 @@ public class DocumentArchivingConfiguration {
@Bean
@StepScope
public DocumentExporter documentExporter(DocumentItemReader documentItemReader,
- @Value("#{stepExecution.jobExecution.id}") String jobId,
+ @Value("#{stepExecution.jobExecution.jobId}") String jobId,
@Value("#{stepExecution.jobExecution.executionContext.get('jobProperties')}") DocumentArchivingProperties properties,
InfraManagerDataConfig infraManagerDataConfig,
@Value("#{jobParameters[end]}") String intervalEnd,
- DocumentWiper documentWiper) {
+ DocumentWiper documentWiper,
+ JobContextRepository jobContextRepository,
+ PasswordStore passwordStore) {
File baseDir = new File(infraManagerDataConfig.getDataFolder(), "exporting");
CompositeFileAction fileAction = new CompositeFileAction(new TarGzCompressor());
switch (properties.getDestination()) {
case S3:
- fileAction.add(new S3Uploader(properties.s3Properties().orElseThrow(() -> new IllegalStateException("S3 properties are not provided!"))));
+ fileAction.add(new S3Uploader(
+ properties.s3Properties().orElseThrow(() -> new IllegalStateException("S3 properties are not provided!")),
+ passwordStore));
break;
case HDFS:
org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
@@ -134,7 +127,7 @@ public class DocumentArchivingConfiguration {
documentItemReader,
firstDocument -> new LocalDocumentItemWriter(
outFile(properties.getSolr().getCollection(), destinationDirectory, fileNameSuffixFormatter.format(firstDocument)), itemWriterListener),
- properties.getWriteBlockSize());
+ properties.getWriteBlockSize(), jobContextRepository);
}
@Bean
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingProperties.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingProperties.java
index b90402a..b26da36 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingProperties.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingProperties.java
@@ -19,23 +19,14 @@
package org.apache.ambari.infra.job.archive;
import org.apache.ambari.infra.job.JobProperties;
-import org.apache.commons.csv.CSVParser;
-import org.apache.commons.csv.CSVRecord;
import org.springframework.batch.core.JobParameters;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.Iterator;
-import java.util.Map;
import java.util.Optional;
-import java.util.function.Supplier;
import static java.util.Objects.requireNonNull;
import static org.apache.ambari.infra.job.archive.ExportDestination.HDFS;
import static org.apache.ambari.infra.job.archive.ExportDestination.LOCAL;
import static org.apache.ambari.infra.job.archive.ExportDestination.S3;
-import static org.apache.commons.csv.CSVFormat.DEFAULT;
import static org.apache.commons.lang.StringUtils.isBlank;
public class DocumentArchivingProperties extends JobProperties<DocumentArchivingProperties> {
@@ -50,47 +41,12 @@ public class DocumentArchivingProperties extends JobProperties<DocumentArchiving
private String s3KeyPrefix;
private String s3BucketName;
private String s3Endpoint;
- private transient Supplier<Optional<S3Properties>> s3Properties;
private String hdfsEndpoint;
private String hdfsDestinationDirectory;
public DocumentArchivingProperties() {
super(DocumentArchivingProperties.class);
- s3Properties = this::loadS3Properties;
- }
-
- private Optional<S3Properties> loadS3Properties() {
- if (isBlank(s3BucketName))
- return Optional.empty();
-
- String accessKey = System.getenv("AWS_ACCESS_KEY_ID");
- String secretKey = System.getenv("AWS_SECRET_ACCESS_KEY");
-
- if (isBlank(accessKey) || isBlank(secretKey)) {
- if (isBlank(s3AccessFile))
- return Optional.empty();
- try (CSVParser csvParser = CSVParser.parse(new FileReader(s3AccessFile), DEFAULT.withHeader("Access key ID", "Secret access key"))) {
- Iterator<CSVRecord> iterator = csvParser.iterator();
- if (!iterator.hasNext()) {
- return Optional.empty();
- }
-
- CSVRecord record = csvParser.iterator().next();
- Map<String, Integer> header = csvParser.getHeaderMap();
- accessKey = record.get(header.get("Access key ID"));
- secretKey = record.get(header.get("Secret access key"));
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- return Optional.of(new S3Properties(
- accessKey,
- secretKey,
- s3KeyPrefix,
- s3BucketName,
- s3Endpoint));
}
public int getReadBlockSize() {
@@ -155,7 +111,6 @@ public class DocumentArchivingProperties extends JobProperties<DocumentArchiving
public void setS3AccessFile(String s3AccessFile) {
this.s3AccessFile = s3AccessFile;
- s3Properties = this::loadS3Properties;
}
public String getS3KeyPrefix() {
@@ -183,7 +138,14 @@ public class DocumentArchivingProperties extends JobProperties<DocumentArchiving
}
public Optional<S3Properties> s3Properties() {
- return s3Properties.get();
+ if (isBlank(s3BucketName))
+ return Optional.empty();
+
+ return Optional.of(new S3Properties(
+ s3AccessFile,
+ s3KeyPrefix,
+ s3BucketName,
+ s3Endpoint));
}
public String getHdfsEndpoint() {
@@ -259,6 +221,7 @@ public class DocumentArchivingProperties extends JobProperties<DocumentArchiving
"The property hdfsDestinationDirectory can not be null or empty string when destination is set to %s!", HDFS.name()));
}
+ requireNonNull(solr, "No solr query was specified for archiving job!");
solr.validate();
}
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingPropertyMap.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingPropertyMap.java
index 7d1a738..a009031 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingPropertyMap.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingPropertyMap.java
@@ -18,7 +18,6 @@
*/
package org.apache.ambari.infra.job.archive;
-import org.apache.ambari.infra.job.PropertyMap;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@@ -26,7 +25,7 @@ import java.util.Map;
@Configuration
@ConfigurationProperties(prefix = "infra-manager.jobs")
-public class DocumentArchivingPropertyMap implements PropertyMap<DocumentArchivingProperties> {
+public class DocumentArchivingPropertyMap {
private Map<String, DocumentArchivingProperties> solrDataArchiving;
public Map<String, DocumentArchivingProperties> getSolrDataArchiving() {
@@ -36,9 +35,4 @@ public class DocumentArchivingPropertyMap implements PropertyMap<DocumentArchivi
public void setSolrDataArchiving(Map<String, DocumentArchivingProperties> solrDataArchiving) {
this.solrDataArchiving = solrDataArchiving;
}
-
- @Override
- public Map<String, DocumentArchivingProperties> getPropertyMap() {
- return getSolrDataArchiving();
- }
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExporter.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExporter.java
index 6106c20..d87fdea 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExporter.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentExporter.java
@@ -18,6 +18,10 @@
*/
package org.apache.ambari.infra.job.archive;
+import org.apache.ambari.infra.job.JobContextRepository;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.ExitStatus;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.StepExecution;
@@ -30,15 +34,19 @@ import org.springframework.batch.repeat.RepeatStatus;
public class DocumentExporter implements Tasklet, StepExecutionListener {
+ private static final Logger LOG = LoggerFactory.getLogger(DocumentExporter.class);
+
private boolean complete = false;
private final ItemStreamReader<Document> documentReader;
private final DocumentDestination documentDestination;
private final int writeBlockSize;
+ private final JobContextRepository jobContextRepository;
- public DocumentExporter(ItemStreamReader<Document> documentReader, DocumentDestination documentDestination, int writeBlockSize) {
+ public DocumentExporter(ItemStreamReader<Document> documentReader, DocumentDestination documentDestination, int writeBlockSize, JobContextRepository jobContextRepository) {
this.documentReader = documentReader;
this.documentDestination = documentDestination;
this.writeBlockSize = writeBlockSize;
+ this.jobContextRepository = jobContextRepository;
}
@Override
@@ -58,7 +66,8 @@ public class DocumentExporter implements Tasklet, StepExecutionListener {
@Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
- ExecutionContext executionContext = chunkContext.getStepContext().getStepExecution().getExecutionContext();
+ StepExecution stepExecution = chunkContext.getStepContext().getStepExecution();
+ ExecutionContext executionContext = stepExecution.getExecutionContext();
documentReader.open(executionContext);
DocumentItemWriter writer = null;
@@ -67,10 +76,19 @@ public class DocumentExporter implements Tasklet, StepExecutionListener {
Document document;
while ((document = documentReader.read()) != null) {
if (writer != null && writtenCount >= writeBlockSize) {
+ stepExecution = jobContextRepository.getStepExecution(stepExecution.getJobExecutionId(), stepExecution.getId());
+ if (stepExecution.getJobExecution().getStatus() == BatchStatus.STOPPING) {
+ LOG.info("Received stop signal.");
+ writer.revert();
+ writer = null;
+ return RepeatStatus.CONTINUABLE;
+ }
+
writer.close();
writer = null;
writtenCount = 0;
documentReader.update(executionContext);
+ jobContextRepository.updateExecutionContext(stepExecution);
}
if (writer == null)
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/FileNameSuffixFormatter.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/FileNameSuffixFormatter.java
index 85ec00b..f9016e6 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/FileNameSuffixFormatter.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/FileNameSuffixFormatter.java
@@ -26,7 +26,7 @@ import static org.apache.ambari.infra.job.archive.SolrDocumentIterator.SOLR_DATE
import static org.apache.commons.lang.StringUtils.isBlank;
public class FileNameSuffixFormatter {
- private static final DateTimeFormatter SOLR_DATETIME_FORMATTER = DateTimeFormatter.ofPattern(SOLR_DATE_FORMAT_TEXT);
+ public static final DateTimeFormatter SOLR_DATETIME_FORMATTER = DateTimeFormatter.ofPattern(SOLR_DATE_FORMAT_TEXT);
public static FileNameSuffixFormatter from(DocumentArchivingProperties properties) {
return new FileNameSuffixFormatter(properties.getFileNameSuffixColumn(), properties.getFileNameSuffixDateFormat());
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java
new file mode 100644
index 0000000..879b58b
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job.archive;
+
+import org.apache.ambari.infra.conf.security.PasswordStore;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.UncheckedIOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.apache.commons.csv.CSVFormat.DEFAULT;
+
+public class S3AccessCsv implements PasswordStore {
+ private static final Logger LOG = LoggerFactory.getLogger(S3AccessCsv.class);
+
+ public static S3AccessCsv file(String path) {
+ try {
+ return new S3AccessCsv(new FileReader(path));
+ } catch (FileNotFoundException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private Map<String, String> passwordMap = new HashMap<>();
+
+ public S3AccessCsv(Reader reader) {
+ try (CSVParser csvParser = CSVParser.parse(reader, DEFAULT.withHeader(
+ S3AccessKeyNames.AccessKeyId.getCsvName(), S3AccessKeyNames.SecretAccessKey.getCsvName()))) {
+ Iterator<CSVRecord> iterator = csvParser.iterator();
+ if (!iterator.hasNext()) {
+ throw new S3AccessCsvFormatException("Csv file is empty!");
+ }
+
+ CSVRecord record = iterator.next();
+ if (record.size() < 2) {
+ throw new S3AccessCsvFormatException("Csv file contains less than 2 columns!");
+ }
+
+ checkColumnExists(record, S3AccessKeyNames.AccessKeyId);
+ checkColumnExists(record, S3AccessKeyNames.SecretAccessKey);
+
+ if (!iterator.hasNext()) {
+ throw new S3AccessCsvFormatException("Csv file contains header only!");
+ }
+
+ record = iterator.next();
+
+ Map<String, Integer> header = csvParser.getHeaderMap();
+ for (S3AccessKeyNames keyNames : S3AccessKeyNames.values())
+ passwordMap.put(keyNames.getEnvVariableName(), record.get(header.get(keyNames.getCsvName())));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ } catch (S3AccessCsvFormatException e) {
+ LOG.warn("Unable to parse csv file: {}", e.getMessage());
+ }
+ }
+
+ private void checkColumnExists(CSVRecord record, S3AccessKeyNames s3AccessKeyName) {
+ if (!s3AccessKeyName.getCsvName().equals(record.get(s3AccessKeyName.getCsvName()))) {
+ throw new S3AccessCsvFormatException(String.format("Csv file does not contain the required column: '%s'", s3AccessKeyName.getCsvName()));
+ }
+ }
+
+ @Override
+ public Optional<String> getPassword(String propertyName) {
+ return Optional.ofNullable(passwordMap.get(propertyName));
+ }
+}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsvFormatException.java
similarity index 80%
copy from ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
copy to ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsvFormatException.java
index e7b9e77..ef9d539 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/PropertyMap.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsvFormatException.java
@@ -16,10 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.infra.job;
+package org.apache.ambari.infra.job.archive;
-import java.util.Map;
-
-public interface PropertyMap<T extends JobProperties<T>> {
- Map<String, T> getPropertyMap();
+public class S3AccessCsvFormatException extends RuntimeException {
+ public S3AccessCsvFormatException(String message) {
+ super(message);
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditBarGraphRequest.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessKeyNames.java
similarity index 60%
copy from ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditBarGraphRequest.java
copy to ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessKeyNames.java
index 03ca32d..e840d3b 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditBarGraphRequest.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessKeyNames.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -16,25 +16,25 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.logsearch.model.request.impl;
+package org.apache.ambari.infra.job.archive;
-import org.apache.ambari.logsearch.common.LogSearchConstants;
-import org.apache.ambari.logsearch.model.request.UnitParamDefinition;
+public enum S3AccessKeyNames {
+ AccessKeyId("AWS_ACCESS_KEY_ID", "Access key ID"),
+ SecretAccessKey("AWS_SECRET_ACCESS_KEY", "Secret access key");
-import javax.ws.rs.QueryParam;
+ private final String envVariableName;
+ private final String csvName;
-public class AuditBarGraphRequest extends BaseLogRequest implements UnitParamDefinition {
-
- @QueryParam(LogSearchConstants.REQUEST_PARAM_UNIT)
- private String unit;
+ S3AccessKeyNames(String envVariableName, String csvName) {
+ this.envVariableName = envVariableName;
+ this.csvName = csvName;
+ }
- @Override
- public String getUnit() {
- return unit;
+ public String getEnvVariableName() {
+ return envVariableName;
}
- @Override
- public void setUnit(String unit) {
- this.unit = unit;
+ public String getCsvName() {
+ return csvName;
}
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Properties.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Properties.java
index 88b71cf..59a4469 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Properties.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Properties.java
@@ -21,28 +21,18 @@ package org.apache.ambari.infra.job.archive;
import static org.apache.commons.lang.StringUtils.isBlank;
public class S3Properties {
- private final String s3AccessKey;
- private final String s3SecretKey;
+ private final String s3AccessFile;
private final String s3KeyPrefix;
private final String s3BucketName;
private final String s3EndPoint;
- public S3Properties(String s3AccessKey, String s3SecretKey, String s3KeyPrefix, String s3BucketName, String s3EndPoint) {
- this.s3AccessKey = s3AccessKey;
- this.s3SecretKey = s3SecretKey;
+ public S3Properties(String s3AccessFile, String s3KeyPrefix, String s3BucketName, String s3EndPoint) {
+ this.s3AccessFile = s3AccessFile;
this.s3KeyPrefix = s3KeyPrefix;
this.s3BucketName = s3BucketName;
this.s3EndPoint = s3EndPoint;
}
- public String getS3AccessKey() {
- return s3AccessKey;
- }
-
- public String getS3SecretKey() {
- return s3SecretKey;
- }
-
public String getS3KeyPrefix() {
return s3KeyPrefix;
}
@@ -55,10 +45,14 @@ public class S3Properties {
return s3EndPoint;
}
+ public String getS3AccessFile() {
+ return s3AccessFile;
+ }
+
@Override
public String toString() {
return "S3Properties{" +
- "s3AccessKey='" + s3AccessKey + '\'' +
+ "s3AccessFile='" + s3AccessFile + '\'' +
", s3KeyPrefix='" + s3KeyPrefix + '\'' +
", s3BucketName='" + s3BucketName + '\'' +
", s3EndPoint='" + s3EndPoint + '\'' +
@@ -66,12 +60,6 @@ public class S3Properties {
}
public void validate() {
- if (isBlank(s3AccessKey))
- throw new IllegalArgumentException("The property s3AccessKey can not be null or empty string!");
-
- if (isBlank(s3SecretKey))
- throw new IllegalArgumentException("The property s3SecretKey can not be null or empty string!");
-
if (isBlank(s3BucketName))
throw new IllegalArgumentException("The property s3BucketName can not be null or empty string!");
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
index 0ab68ed..2536cb5 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
@@ -2,12 +2,15 @@ package org.apache.ambari.infra.job.archive;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
+import org.apache.ambari.infra.conf.security.CompositePasswordStore;
+import org.apache.ambari.infra.conf.security.PasswordStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import static org.apache.commons.lang.StringUtils.isBlank;
+import static org.apache.commons.lang.StringUtils.isNotBlank;
/*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -35,12 +38,21 @@ public class S3Uploader extends AbstractFileAction {
private final String keyPrefix;
private final String bucketName;
- public S3Uploader(S3Properties s3Properties) {
+ public S3Uploader(S3Properties s3Properties, PasswordStore passwordStore) {
LOG.info("Initializing S3 client with " + s3Properties);
this.keyPrefix = s3Properties.getS3KeyPrefix();
this.bucketName = s3Properties.getS3BucketName();
- BasicAWSCredentials credentials = new BasicAWSCredentials(s3Properties.getS3AccessKey(), s3Properties.getS3SecretKey());
+
+ PasswordStore compositePasswordStore = passwordStore;
+ if (isNotBlank((s3Properties.getS3AccessFile())))
+ compositePasswordStore = new CompositePasswordStore(passwordStore, S3AccessCsv.file(s3Properties.getS3AccessFile()));
+
+ BasicAWSCredentials credentials = new BasicAWSCredentials(
+ compositePasswordStore.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName())
+ .orElseThrow(() -> new IllegalArgumentException("Access key Id is not present!")),
+ compositePasswordStore.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName())
+ .orElseThrow(() -> new IllegalArgumentException("Secret Access Key is not present!")));
client = new AmazonS3Client(credentials);
if (!isBlank(s3Properties.getS3EndPoint()))
client.setEndpoint(s3Properties.getS3EndPoint());
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingConfiguration.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingConfiguration.java
index ce8970d..4a68c49 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingConfiguration.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingConfiguration.java
@@ -18,9 +18,8 @@
*/
package org.apache.ambari.infra.job.deleting;
-import org.apache.ambari.infra.job.JobPropertyMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.ambari.infra.job.AbstractJobsConfiguration;
+import org.apache.ambari.infra.job.JobScheduler;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
@@ -28,50 +27,36 @@ import org.springframework.batch.core.configuration.annotation.JobScope;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.configuration.support.JobRegistryBeanPostProcessor;
+import org.springframework.batch.core.job.builder.JobBuilder;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
-import javax.annotation.PostConstruct;
import javax.inject.Inject;
@Configuration
-public class DocumentDeletingConfiguration {
- private static final Logger LOG = LoggerFactory.getLogger(DocumentDeletingConfiguration.class);
+public class DocumentDeletingConfiguration extends AbstractJobsConfiguration<DocumentDeletingProperties> {
- @Inject
- private DocumentDeletingPropertyMap propertyMap;
-
- @Inject
- private StepBuilderFactory steps;
-
- @Inject
- private JobBuilderFactory jobs;
+ private final StepBuilderFactory steps;
+ private final Step deleteStep;
@Inject
- private JobRegistryBeanPostProcessor jobRegistryBeanPostProcessor;
-
- @Inject
- @Qualifier("deleteStep")
- private Step deleteStep;
-
- @PostConstruct
- public void createJobs() {
- if (propertyMap == null || propertyMap.getSolrDataDeleting() == null)
- return;
-
- propertyMap.getSolrDataDeleting().values().forEach(DocumentDeletingProperties::validate);
-
- propertyMap.getSolrDataDeleting().keySet().forEach(jobName -> {
- LOG.info("Registering data deleting job {}", jobName);
- Job job = logDeleteJob(jobName, deleteStep);
- jobRegistryBeanPostProcessor.postProcessAfterInitialization(job, jobName);
- });
+ public DocumentDeletingConfiguration(
+ DocumentDeletingPropertyMap documentDeletingPropertyMap,
+ JobScheduler scheduler,
+ StepBuilderFactory steps,
+ JobBuilderFactory jobs,
+ JobRegistryBeanPostProcessor jobRegistryBeanPostProcessor,
+ @Qualifier("deleteStep") Step deleteStep) {
+ super(documentDeletingPropertyMap.getSolrDataDeleting(), scheduler, jobs, jobRegistryBeanPostProcessor);
+ this.steps = steps;
+ this.deleteStep = deleteStep;
}
- private Job logDeleteJob(String jobName, Step logExportStep) {
- return jobs.get(jobName).listener(new JobPropertyMap<>(propertyMap)).start(logExportStep).build();
+ @Override
+ protected Job buildJob(JobBuilder jobBuilder) {
+ return jobBuilder.start(deleteStep).build();
}
@Bean
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingPropertyMap.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingPropertyMap.java
index fccfd59..1dc0caf 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingPropertyMap.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingPropertyMap.java
@@ -18,7 +18,6 @@
*/
package org.apache.ambari.infra.job.deleting;
-import org.apache.ambari.infra.job.PropertyMap;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@@ -26,7 +25,7 @@ import java.util.Map;
@Configuration
@ConfigurationProperties(prefix = "infra-manager.jobs")
-public class DocumentDeletingPropertyMap implements PropertyMap<DocumentDeletingProperties> {
+public class DocumentDeletingPropertyMap {
private Map<String, DocumentDeletingProperties> solrDataDeleting;
public Map<String, DocumentDeletingProperties> getSolrDataDeleting() {
@@ -36,9 +35,4 @@ public class DocumentDeletingPropertyMap implements PropertyMap<DocumentDeleting
public void setSolrDataDeleting(Map<String, DocumentDeletingProperties> solrDataDeleting) {
this.solrDataDeleting = solrDataDeleting;
}
-
- @Override
- public Map<String, DocumentDeletingProperties> getPropertyMap() {
- return getSolrDataDeleting();
- }
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/manager/JobManager.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/manager/JobManager.java
index 862119a..f35387d 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/manager/JobManager.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/manager/JobManager.java
@@ -18,7 +18,6 @@
*/
package org.apache.ambari.infra.manager;
-import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import org.apache.ambari.infra.model.ExecutionContextResponse;
import org.apache.ambari.infra.model.JobDetailsResponse;
@@ -36,9 +35,13 @@ import org.springframework.batch.admin.service.JobService;
import org.springframework.batch.admin.service.NoSuchStepExecutionException;
import org.springframework.batch.admin.web.JobInfo;
import org.springframework.batch.admin.web.StepExecutionProgress;
-import org.springframework.batch.core.*;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.core.JobInstance;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.batch.core.JobParametersInvalidException;
+import org.springframework.batch.core.StepExecution;
+import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.launch.JobExecutionNotRunningException;
-import org.springframework.batch.core.launch.JobInstanceAlreadyExistsException;
import org.springframework.batch.core.launch.JobOperator;
import org.springframework.batch.core.launch.NoSuchJobException;
import org.springframework.batch.core.launch.NoSuchJobExecutionException;
@@ -51,16 +54,16 @@ import javax.inject.Inject;
import javax.inject.Named;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.TimeZone;
@Named
-public class JobManager {
+public class JobManager implements Jobs {
private static final Logger LOG = LoggerFactory.getLogger(JobManager.class);
@@ -70,6 +73,9 @@ public class JobManager {
@Inject
private JobOperator jobOperator;
+ @Inject
+ private JobExplorer jobExplorer;
+
private TimeZone timeZone = TimeZone.getDefault();
public Set<String> getAllJobNames() {
@@ -80,18 +86,28 @@ public class JobManager {
* Launch a new job instance (based on job name) and applies customized parameters to it.
* Also add a new date parameter to make sure the job instance will be unique
*/
- public JobExecutionInfoResponse launchJob(String jobName, String params)
- throws JobParametersInvalidException, JobInstanceAlreadyExistsException, NoSuchJobException,
+ @Override
+ public JobExecutionInfoResponse launchJob(String jobName, JobParameters jobParameters)
+ throws JobParametersInvalidException, NoSuchJobException,
JobExecutionAlreadyRunningException, JobRestartException, JobInstanceAlreadyCompleteException {
- JobParametersBuilder jobParametersBuilder = new JobParametersBuilder();
- if (params != null) {
- LOG.info("Parsing parameters of job {} '{}'", jobName, params);
- Splitter.on(',')
- .trimResults()
- .withKeyValueSeparator(Splitter.on('=').limit(2).trimResults())
- .split(params).entrySet().forEach(entry -> jobParametersBuilder.addString(entry.getKey(), entry.getValue()));
- }
- return new JobExecutionInfoResponse(jobService.launch(jobName, jobParametersBuilder.toJobParameters()), timeZone);
+
+ Set<JobExecution> running = jobExplorer.findRunningJobExecutions(jobName);
+ if (!running.isEmpty())
+ throw new JobExecutionAlreadyRunningException("An instance of this job is already active: "+jobName);
+
+ return new JobExecutionInfoResponse(jobService.launch(jobName, jobParameters), timeZone);
+ }
+
+ @Override
+ public void restart(Long jobExecutionId)
+ throws JobInstanceAlreadyCompleteException, NoSuchJobException, JobExecutionAlreadyRunningException,
+ JobParametersInvalidException, JobRestartException, NoSuchJobExecutionException {
+ jobService.restart(jobExecutionId);
+ }
+
+ @Override
+ public Optional<JobExecution> lastRun(String jobName) throws NoSuchJobException {
+ return jobService.listJobExecutionsForJob(jobName, 0, 1).stream().findFirst();
}
/**
@@ -111,19 +127,14 @@ public class JobManager {
/**
* Gather job execution details by job execution id.
*/
- public JobExecutionDetailsResponse getExectionInfo(Long jobExecutionId) throws NoSuchJobExecutionException {
+ public JobExecutionDetailsResponse getExecutionInfo(Long jobExecutionId) throws NoSuchJobExecutionException {
JobExecution jobExecution = jobService.getJobExecution(jobExecutionId);
- List<StepExecutionInfoResponse> stepExecutionInfos = new ArrayList<StepExecutionInfoResponse>();
+ List<StepExecutionInfoResponse> stepExecutionInfoList = new ArrayList<>();
for (StepExecution stepExecution : jobExecution.getStepExecutions()) {
- stepExecutionInfos.add(new StepExecutionInfoResponse(stepExecution, timeZone));
+ stepExecutionInfoList.add(new StepExecutionInfoResponse(stepExecution, timeZone));
}
- Collections.sort(stepExecutionInfos, new Comparator<StepExecutionInfoResponse>() {
- @Override
- public int compare(StepExecutionInfoResponse o1, StepExecutionInfoResponse o2) {
- return o1.getId().compareTo(o2.getId());
- }
- });
- return new JobExecutionDetailsResponse(new JobExecutionInfoResponse(jobExecution, timeZone), stepExecutionInfos);
+ stepExecutionInfoList.sort(Comparator.comparing(StepExecutionInfoResponse::getId));
+ return new JobExecutionDetailsResponse(new JobExecutionInfoResponse(jobExecution, timeZone), stepExecutionInfoList);
}
/**
@@ -139,6 +150,7 @@ public class JobManager {
} else {
throw new UnsupportedOperationException("Unsupported operaration");
}
+ LOG.info("Job {} was marked {}", jobExecution.getJobInstance().getJobName(), operation.name());
return new JobExecutionInfoResponse(jobExecution, timeZone);
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/manager/Jobs.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/manager/Jobs.java
new file mode 100644
index 0000000..b2ca605
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/manager/Jobs.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.manager;
+
+import org.apache.ambari.infra.model.JobExecutionInfoResponse;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.batch.core.JobParametersInvalidException;
+import org.springframework.batch.core.launch.NoSuchJobException;
+import org.springframework.batch.core.launch.NoSuchJobExecutionException;
+import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
+import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
+import org.springframework.batch.core.repository.JobRestartException;
+
+import java.util.Optional;
+
+public interface Jobs {
+ JobExecutionInfoResponse launchJob(String jobName, JobParameters params)
+ throws JobParametersInvalidException, NoSuchJobException,
+ JobExecutionAlreadyRunningException, JobRestartException, JobInstanceAlreadyCompleteException;
+ void restart(Long jobExecutionId)
+ throws JobInstanceAlreadyCompleteException, NoSuchJobException, JobExecutionAlreadyRunningException,
+ JobParametersInvalidException, JobRestartException, NoSuchJobExecutionException;
+
+ Optional<JobExecution> lastRun(String jobName) throws NoSuchJobException, NoSuchJobExecutionException;
+}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/rest/JobResource.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/rest/JobResource.java
index 0e20b54..502057e 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/rest/JobResource.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/rest/JobResource.java
@@ -18,6 +18,7 @@
*/
package org.apache.ambari.infra.rest;
+import com.google.common.base.Splitter;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.apache.ambari.infra.manager.JobManager;
@@ -35,11 +36,13 @@ import org.apache.ambari.infra.model.StepExecutionContextResponse;
import org.apache.ambari.infra.model.StepExecutionInfoResponse;
import org.apache.ambari.infra.model.StepExecutionProgressResponse;
import org.apache.ambari.infra.model.StepExecutionRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.batch.admin.service.NoSuchStepExecutionException;
import org.springframework.batch.admin.web.JobInfo;
+import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.batch.core.launch.JobExecutionNotRunningException;
-import org.springframework.batch.core.launch.JobInstanceAlreadyExistsException;
import org.springframework.batch.core.launch.NoSuchJobException;
import org.springframework.batch.core.launch.NoSuchJobExecutionException;
import org.springframework.batch.core.launch.NoSuchJobInstanceException;
@@ -67,6 +70,7 @@ import java.util.Set;
@Named
@Scope("request")
public class JobResource {
+ private static final Logger LOG = LoggerFactory.getLogger(JobResource.class);
@Inject
private JobManager jobManager;
@@ -83,9 +87,21 @@ public class JobResource {
@Path("{jobName}")
@ApiOperation("Start a new job instance by job name.")
public JobExecutionInfoResponse startJob(@BeanParam @Valid JobInstanceStartRequest request)
- throws JobParametersInvalidException, JobInstanceAlreadyExistsException, NoSuchJobException, JobExecutionAlreadyRunningException,
+ throws JobParametersInvalidException, NoSuchJobException, JobExecutionAlreadyRunningException,
JobRestartException, JobInstanceAlreadyCompleteException {
- return jobManager.launchJob(request.getJobName(), request.getParams());
+
+ String jobName = request.getJobName();
+ String params = request.getParams();
+ JobParametersBuilder jobParametersBuilder = new JobParametersBuilder();
+ if (params != null) {
+ LOG.info("Parsing parameters of job {} '{}'", jobName, params);
+ Splitter.on(',')
+ .trimResults()
+ .withKeyValueSeparator(Splitter.on('=').limit(2).trimResults())
+ .split(params).forEach(jobParametersBuilder::addString);
+ }
+
+ return jobManager.launchJob(jobName, jobParametersBuilder.toJobParameters());
}
@GET
@@ -117,7 +133,7 @@ public class JobResource {
@Path("/executions/{jobExecutionId}")
@ApiOperation("Get job and step details for job execution instance.")
public JobExecutionDetailsResponse getExectionInfo(@PathParam("jobExecutionId") @Valid Long jobExecutionId) throws NoSuchJobExecutionException {
- return jobManager.getExectionInfo(jobExecutionId);
+ return jobManager.getExecutionInfo(jobExecutionId);
}
@GET
@@ -150,8 +166,8 @@ public class JobResource {
@Produces({"application/json"})
@Path("/{jobName}/{jobInstanceId}/executions")
@ApiOperation("Get execution for job instance.")
- public List<JobExecutionInfoResponse> getExecutionsForInstance(@BeanParam @Valid JobExecutionRequest request) throws JobInstanceAlreadyCompleteException,
- NoSuchJobExecutionException, JobExecutionAlreadyRunningException, JobParametersInvalidException, JobRestartException, NoSuchJobException, NoSuchJobInstanceException {
+ public List<JobExecutionInfoResponse> getExecutionsForInstance(@BeanParam @Valid JobExecutionRequest request) throws
+ NoSuchJobException, NoSuchJobInstanceException {
return jobManager.getExecutionsForJobInstance(request.getJobName(), request.getJobInstanceId());
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties b/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties
index aea2b88..a0712ba 100644
--- a/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties
+++ b/ambari-infra/ambari-infra-manager/src/main/resources/infra-manager.properties
@@ -14,13 +14,14 @@
# limitations under the License.
infra-manager.batch.db.file=job-repository.db
-infra-manager.batch.db.init=true
+infra-manager.batch.db.init=false
infra-manager.batch.db.username=admin
infra-manager.batch.db.password=admin
management.security.enabled=false
management.health.solr.enabled=false
infra-manager.server.data.folder=/tmp/ambariInfraManager
+infra-manager.jobs.solr_data_archiving.archive_service_logs.enabled=true
infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.zoo_keeper_connection_string=zookeeper:2181
infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.collection=hadoop_logs
infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.query_text=logtime:[${start} TO ${end}]
@@ -33,6 +34,10 @@ infra-manager.jobs.solr_data_archiving.archive_service_logs.destination=LOCAL
infra-manager.jobs.solr_data_archiving.archive_service_logs.local_destination_directory=/tmp/ambariInfraManager
infra-manager.jobs.solr_data_archiving.archive_service_logs.file_name_suffix_column=logtime
infra-manager.jobs.solr_data_archiving.archive_service_logs.file_name_suffix_date_format=yyyy-MM-dd'T'HH-mm-ss.SSSX
+infra-manager.jobs.solr_data_archiving.archive_service_logs.scheduling.enabled=true
+infra-manager.jobs.solr_data_archiving.archive_service_logs.scheduling.cron=0 * * * * ?
+infra-manager.jobs.solr_data_archiving.archive_service_logs.scheduling.intervalEndDelta=PT24H
+infra-manager.jobs.solr_data_archiving.archive_audit_logs.enabled=true
infra-manager.jobs.solr_data_archiving.archive_audit_logs.solr.zoo_keeper_connection_string=zookeeper:2181
infra-manager.jobs.solr_data_archiving.archive_audit_logs.solr.collection=audit_logs
infra-manager.jobs.solr_data_archiving.archive_audit_logs.solr.query_text=logtime:[${start} TO ${end}]
@@ -63,6 +68,7 @@ infra-manager.jobs.solr_data_archiving.archive_audit_logs.s3_endpoint=http://fak
#infra-manager.jobs.solr_data_archiving.export_ranger_audit_logs.query.filter_query_text=(logtime:"${logtime}" AND id:{"${id}" TO *]) OR logtime:{"${logtime}" TO "${end}"]
#infra-manager.jobs.solr_data_archiving.export_ranger_audit_logs.query.sort_column[0]=logtime
#infra-manager.jobs.solr_data_archiving.export_ranger_audit_logs.query.sort_column[1]=id
+infra-manager.jobs.solr_data_deleting.delete_audit_logs.enabled=true
infra-manager.jobs.solr_data_deleting.delete_audit_logs.zoo_keeper_connection_string=zookeeper:2181
infra-manager.jobs.solr_data_deleting.delete_audit_logs.collection=audit_logs
infra-manager.jobs.solr_data_deleting.delete_audit_logs.filter_field=logtime
diff --git a/ambari-infra/ambari-infra-manager/src/main/resources/log4j2.xml b/ambari-infra/ambari-infra-manager/src/main/resources/log4j2.xml
index 9737554..d3db3d7 100644
--- a/ambari-infra/ambari-infra-manager/src/main/resources/log4j2.xml
+++ b/ambari-infra/ambari-infra-manager/src/main/resources/log4j2.xml
@@ -37,5 +37,8 @@
<AppenderRef ref="File" />
<AppenderRef ref="Console" />
</Root>
+ <!--<Logger name="org.springframework.jdbc.core.JdbcTemplate" level="debug">-->
+ <!--<AppenderRef ref="Console"/>-->
+ <!--</Logger>-->
</Loggers>
</Configuration>
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java
new file mode 100644
index 0000000..26a6953
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java
@@ -0,0 +1,48 @@
+package org.apache.ambari.infra.conf.security;
+
+import org.junit.Test;
+
+import java.util.Optional;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+public class CompositePasswordStoreTest {
+ @Test
+ public void testGetPasswordReturnNullIfNoPasswordStoresWereAdded() {
+ assertThat(new CompositePasswordStore().getPassword("any").isPresent(), is(false));
+ }
+
+ @Test
+ public void testGetPasswordReturnNullIfPasswordNotFoundInAnyStore() {
+ assertThat(new CompositePasswordStore((prop) -> Optional.empty(), (prop) -> Optional.empty()).getPassword("any").isPresent(), is(false));
+ }
+
+ @Test
+ public void testGetPasswordReturnPasswordFromFirstStoreIfExists() {
+ assertThat(new CompositePasswordStore((prop) -> Optional.of("Pass"), (prop) -> Optional.empty()).getPassword("any").get(), is("Pass"));
+ }
+
+ @Test
+ public void testGetPasswordReturnPasswordFromSecondStoreIfNotExistsInFirst() {
+ assertThat(new CompositePasswordStore((prop) -> Optional.empty(), (prop) -> Optional.of("Pass")).getPassword("any").get(), is("Pass"));
+ }
+}
\ No newline at end of file
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/JobSchedulerTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/JobSchedulerTest.java
new file mode 100644
index 0000000..ba1150f
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/JobSchedulerTest.java
@@ -0,0 +1,114 @@
+package org.apache.ambari.infra.job;
+
+import org.apache.ambari.infra.manager.Jobs;
+import org.easymock.EasyMockRunner;
+import org.easymock.EasyMockSupport;
+import org.easymock.Mock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.batch.core.ExitStatus;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.scheduling.support.CronTrigger;
+
+import javax.batch.operations.NoSuchJobException;
+import java.util.Optional;
+import java.util.concurrent.ScheduledFuture;
+
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.isA;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+@RunWith(EasyMockRunner.class)
+public class JobSchedulerTest extends EasyMockSupport {
+
+ @Mock
+ private TaskScheduler taskScheduler;
+ @Mock
+ private Jobs jobs;
+ @Mock
+ private ScheduledFuture scheduledFuture;
+ private JobScheduler jobScheduler;
+
+ @Before
+ public void setUp() throws Exception {
+ jobScheduler = new JobScheduler(taskScheduler, jobs);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ verifyAll();
+ }
+
+ @Test(expected = NoSuchJobException.class)
+ public void testScheduleWhenJobNotExistsThrowsException() throws Exception {
+ String jobName = "notFoundJob";
+ expect(jobs.lastRun(jobName)).andThrow(new NoSuchJobException());
+ replayAll();
+
+ jobScheduler.schedule(jobName, null);
+ }
+
+ @Test
+ public void testScheduleWhenNoPreviousExecutionExistsJobIsScheduled() throws Exception {
+ String jobName = "job0";
+ SchedulingProperties schedulingProperties = new SchedulingProperties();
+ schedulingProperties.setCron("* * * * * ?");
+ expect(jobs.lastRun(jobName)).andReturn(Optional.empty());
+ expect(taskScheduler.schedule(isA(Runnable.class), eq(new CronTrigger(schedulingProperties.getCron())))).andReturn(scheduledFuture);
+ replayAll();
+
+ jobScheduler.schedule(jobName, schedulingProperties);
+ }
+
+ @Test
+ public void testScheduleWhenPreviousExecutionWasSuccessfulJobIsScheduled() throws Exception {
+ String jobName = "job0";
+ SchedulingProperties schedulingProperties = new SchedulingProperties();
+ schedulingProperties.setCron("* * * * * ?");
+ JobExecution jobExecution = new JobExecution(1L, new JobParameters());
+ jobExecution.setExitStatus(ExitStatus.COMPLETED);
+ expect(jobs.lastRun(jobName)).andReturn(Optional.of(jobExecution));
+ expect(taskScheduler.schedule(isA(Runnable.class), eq(new CronTrigger(schedulingProperties.getCron())))).andReturn(scheduledFuture);
+ replayAll();
+
+ jobScheduler.schedule(jobName, schedulingProperties);
+ }
+
+ @Test
+ public void testScheduleWhenPreviousExecutionFailedJobIsRestartedAndScheduled() throws Exception {
+ String jobName = "job0";
+ SchedulingProperties schedulingProperties = new SchedulingProperties();
+ schedulingProperties.setCron("* * * * * ?");
+ JobExecution jobExecution = new JobExecution(1L, new JobParameters());
+ jobExecution.setExitStatus(ExitStatus.FAILED);
+ expect(jobs.lastRun(jobName)).andReturn(Optional.of(jobExecution));
+ jobs.restart(1L); expectLastCall();
+ expect(taskScheduler.schedule(isA(Runnable.class), eq(new CronTrigger(schedulingProperties.getCron())))).andReturn(scheduledFuture);
+ replayAll();
+
+ jobScheduler.schedule(jobName, schedulingProperties);
+ }
+}
\ No newline at end of file
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentExporterTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentExporterTest.java
index 88fbff0..b31110c 100644
--- a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentExporterTest.java
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/DocumentExporterTest.java
@@ -19,6 +19,7 @@
package org.apache.ambari.infra.job.archive;
+import org.apache.ambari.infra.job.JobContextRepository;
import org.easymock.EasyMockRunner;
import org.easymock.EasyMockSupport;
import org.easymock.Mock;
@@ -26,12 +27,14 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.scope.context.StepContext;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.item.ItemStreamReader;
+import org.springframework.batch.repeat.RepeatStatus;
import java.io.IOException;
import java.io.UncheckedIOException;
@@ -39,10 +42,20 @@ import java.util.HashMap;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
@RunWith(EasyMockRunner.class)
public class DocumentExporterTest extends EasyMockSupport {
+ private static final long JOB_EXECUTION_ID = 1L;
+ private static final long STEP_EXECUTION_ID = 1L;
+ private static final Document DOCUMENT_2 = new Document(new HashMap<String, String>() {{
+ put("id", "2");
+ }});
+ private static final Document DOCUMENT_3 = new Document(new HashMap<String, String>() {{
+ put("id", "3");
+ }});
private DocumentExporter documentExporter;
@Mock
private ItemStreamReader<Document> reader;
@@ -52,17 +65,30 @@ public class DocumentExporterTest extends EasyMockSupport {
private DocumentItemWriter documentItemWriter;
@Mock
private DocumentItemWriter documentItemWriter2;
+ @Mock
+ private DocumentItemWriter documentItemWriter3;
+ @Mock
+ private JobContextRepository jobContextRepository;
- private ExecutionContext executionContext;
+// private ExecutionContext executionContext;
private ChunkContext chunkContext;
private static final Document DOCUMENT = new Document(new HashMap<String, String>() {{ put("id", "1"); }});
@Before
public void setUp() throws Exception {
- StepExecution stepExecution = new StepExecution("exportDoc", new JobExecution(1L));
- chunkContext = new ChunkContext(new StepContext(stepExecution));
- executionContext = stepExecution.getExecutionContext();
- documentExporter = new DocumentExporter(reader, documentDestination, 2);
+ chunkContext = chunkContext(BatchStatus.STARTED);
+ documentExporter = documentExporter(2);
+ }
+
+ private DocumentExporter documentExporter(int writeBlockSize) {
+ return new DocumentExporter(reader, documentDestination, writeBlockSize, jobContextRepository);
+ }
+
+ private ChunkContext chunkContext(BatchStatus batchStatus) {
+ StepExecution stepExecution = new StepExecution("exportDoc", new JobExecution(JOB_EXECUTION_ID));
+ stepExecution.setId(STEP_EXECUTION_ID);
+ stepExecution.getJobExecution().setStatus(batchStatus);
+ return new ChunkContext(new StepContext(stepExecution));
}
@After
@@ -72,7 +98,7 @@ public class DocumentExporterTest extends EasyMockSupport {
@Test
public void testNothingToRead() throws Exception {
- reader.open(executionContext); expectLastCall();
+ reader.open(executionContext(chunkContext)); expectLastCall();
expect(reader.read()).andReturn(null);
reader.close(); expectLastCall();
replayAll();
@@ -80,9 +106,13 @@ public class DocumentExporterTest extends EasyMockSupport {
documentExporter.execute(null, chunkContext);
}
+ private ExecutionContext executionContext(ChunkContext chunkContext) {
+ return chunkContext.getStepContext().getStepExecution().getExecutionContext();
+ }
+
@Test
public void testWriteLessDocumentsThanWriteBlockSize() throws Exception {
- reader.open(executionContext); expectLastCall();
+ reader.open(executionContext(chunkContext)); expectLastCall();
expect(reader.read()).andReturn(DOCUMENT);
expect(documentDestination.open(DOCUMENT)).andReturn(documentItemWriter);
documentItemWriter.write(DOCUMENT); expectLastCall();
@@ -91,36 +121,35 @@ public class DocumentExporterTest extends EasyMockSupport {
documentItemWriter.close(); expectLastCall();
replayAll();
- documentExporter.execute(null, chunkContext);
+ assertThat(documentExporter.execute(null, chunkContext), is(RepeatStatus.FINISHED));
}
@Test
public void testWriteMoreDocumentsThanWriteBlockSize() throws Exception {
- Document document2 = new Document(new HashMap<String, String>() {{ put("id", "2"); }});
- Document document3 = new Document(new HashMap<String, String>() {{ put("id", "3"); }});
-
- reader.open(executionContext); expectLastCall();
+ reader.open(executionContext(chunkContext)); expectLastCall();
expect(reader.read()).andReturn(DOCUMENT);
expect(documentDestination.open(DOCUMENT)).andReturn(documentItemWriter);
documentItemWriter.write(DOCUMENT); expectLastCall();
- expect(reader.read()).andReturn(document2);
- documentItemWriter.write(document2); expectLastCall();
- expect(reader.read()).andReturn(document3);
+ expect(reader.read()).andReturn(DOCUMENT_2);
+ documentItemWriter.write(DOCUMENT_2); expectLastCall();
+ expect(reader.read()).andReturn(DOCUMENT_3);
documentItemWriter.close(); expectLastCall();
- expect(documentDestination.open(document3)).andReturn(documentItemWriter2);
- documentItemWriter2.write(document3); expectLastCall();
+ jobContextRepository.updateExecutionContext(chunkContext.getStepContext().getStepExecution());
+ expect(jobContextRepository.getStepExecution(JOB_EXECUTION_ID, STEP_EXECUTION_ID)).andReturn(chunkContext.getStepContext().getStepExecution());
+ expect(documentDestination.open(DOCUMENT_3)).andReturn(documentItemWriter2);
+ documentItemWriter2.write(DOCUMENT_3); expectLastCall();
expect(reader.read()).andReturn(null);
- reader.update(executionContext);
+ reader.update(executionContext(chunkContext));
reader.close(); expectLastCall();
documentItemWriter2.close(); expectLastCall();
replayAll();
- documentExporter.execute(null, chunkContext);
+ assertThat(documentExporter.execute(null, chunkContext), is(RepeatStatus.FINISHED));
}
@Test(expected = IOException.class)
public void testReadError() throws Exception {
- reader.open(executionContext); expectLastCall();
+ reader.open(executionContext(chunkContext)); expectLastCall();
expect(reader.read()).andReturn(DOCUMENT);
expect(documentDestination.open(DOCUMENT)).andReturn(documentItemWriter);
documentItemWriter.write(DOCUMENT); expectLastCall();
@@ -134,7 +163,7 @@ public class DocumentExporterTest extends EasyMockSupport {
@Test(expected = UncheckedIOException.class)
public void testWriteError() throws Exception {
- reader.open(executionContext); expectLastCall();
+ reader.open(executionContext(chunkContext)); expectLastCall();
expect(reader.read()).andReturn(DOCUMENT);
expect(documentDestination.open(DOCUMENT)).andReturn(documentItemWriter);
documentItemWriter.write(DOCUMENT); expectLastCall().andThrow(new UncheckedIOException(new IOException("TEST")));
@@ -144,4 +173,43 @@ public class DocumentExporterTest extends EasyMockSupport {
documentExporter.execute(null, chunkContext);
}
+
+ @Test
+ public void testStopAndRestartExportsAllDocuments() throws Exception {
+ ChunkContext stoppingChunkContext = chunkContext(BatchStatus.STOPPING);
+ DocumentExporter documentExporter = documentExporter(1);
+
+ reader.open(executionContext(chunkContext)); expectLastCall();
+ expect(reader.read()).andReturn(DOCUMENT);
+
+ expect(documentDestination.open(DOCUMENT)).andReturn(documentItemWriter);
+ documentItemWriter.write(DOCUMENT); expectLastCall();
+ expect(reader.read()).andReturn(DOCUMENT_2);
+ expect(jobContextRepository.getStepExecution(JOB_EXECUTION_ID, STEP_EXECUTION_ID)).andReturn(chunkContext.getStepContext().getStepExecution());
+ documentItemWriter.close(); expectLastCall();
+ reader.update(executionContext(this.chunkContext));
+ jobContextRepository.updateExecutionContext(this.chunkContext.getStepContext().getStepExecution());
+
+ expect(documentDestination.open(DOCUMENT_2)).andReturn(documentItemWriter2);
+ documentItemWriter2.write(DOCUMENT_2); expectLastCall();
+ expect(reader.read()).andReturn(DOCUMENT_3);
+ expect(jobContextRepository.getStepExecution(JOB_EXECUTION_ID, STEP_EXECUTION_ID)).andReturn(stoppingChunkContext.getStepContext().getStepExecution());
+ documentItemWriter2.revert(); expectLastCall();
+ reader.close(); expectLastCall();
+
+ reader.open(executionContext(chunkContext));
+ expect(reader.read()).andReturn(DOCUMENT_3);
+ expect(documentDestination.open(DOCUMENT_3)).andReturn(documentItemWriter3);
+ documentItemWriter3.write(DOCUMENT_3); expectLastCall();
+ documentItemWriter3.close(); expectLastCall();
+
+ expect(reader.read()).andReturn(null);
+ reader.close(); expectLastCall();
+ replayAll();
+
+ RepeatStatus repeatStatus = documentExporter.execute(null, this.chunkContext);
+ assertThat(repeatStatus, is(RepeatStatus.CONTINUABLE));
+ repeatStatus = documentExporter.execute(null, this.chunkContext);
+ assertThat(repeatStatus, is(RepeatStatus.FINISHED));
+ }
}
\ No newline at end of file
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java
new file mode 100644
index 0000000..e34a222
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java
@@ -0,0 +1,70 @@
+package org.apache.ambari.infra.job.archive;
+
+import org.junit.Test;
+
+import java.io.StringReader;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+public class S3AccessCsvTest {
+
+ private static final String VALID_ACCESS_FILE = "Access key ID,Secret access key\n" +
+ "someKey,someSecret\n";
+
+ private static final String ANY_CSV_FILE = "Column1,Column2\n" +
+ "Foo,Bar\n";
+
+ @Test
+ public void testGetPasswordReturnsNullIfInputIsEmpty() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(""));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ }
+
+ @Test
+ public void testGetPasswordReturnsAccessAndSecretKeyIfInputIsAValidS3AccessFile() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(VALID_ACCESS_FILE));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).get(), is("someKey"));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).get(), is("someSecret"));
+ }
+
+ @Test
+ public void testGetPasswordReturnsNullIfNotAValidS3AccessFileProvided() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(ANY_CSV_FILE));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ }
+
+ @Test
+ public void testGetPasswordReturnsNullIfAHeaderOnlyS3AccessFileProvided() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Secret access key\n"));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ }
+
+ @Test
+ public void testGetPasswordReturnsNullIfOnlyOneValidColumnProvided() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Column\n"));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ }
+}
\ No newline at end of file
diff --git a/ambari-infra/ambari-infra-manager/src/test/resoruces/vagrant-infra-manager.properties.sample b/ambari-infra/ambari-infra-manager/src/test/resoruces/vagrant-infra-manager.properties.sample
index f008a53..d722f0e 100644
--- a/ambari-infra/ambari-infra-manager/src/test/resoruces/vagrant-infra-manager.properties.sample
+++ b/ambari-infra/ambari-infra-manager/src/test/resoruces/vagrant-infra-manager.properties.sample
@@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
infra-manager.batch.db.file=job-repository.db
infra-manager.batch.db.init=true
infra-manager.batch.db.username=admin
@@ -20,18 +21,19 @@ management.security.enabled=false
management.health.solr.enabled=false
infra-manager.server.data.folder=/tmp/ambariInfraManager
-infra-manager.jobs.solr_data_export.archive_service_logs.solr.zoo_keeper_connection_string=c6401.ambari.apache.org:2181/infra-solr
-infra-manager.jobs.solr_data_export.archive_service_logs.solr.collection=hadoop_logs
-infra-manager.jobs.solr_data_export.archive_service_logs.solr.query_text=logtime:[${start} TO ${end}]
-infra-manager.jobs.solr_data_export.archive_service_logs.solr.filter_query_text=(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}]
-infra-manager.jobs.solr_data_export.archive_service_logs.solr.sort_column[0]=logtime
-infra-manager.jobs.solr_data_export.archive_service_logs.solr.sort_column[1]=id
-infra-manager.jobs.solr_data_export.archive_service_logs.solr.delete_query_text=logtime:[${start.logtime} TO ${end.logtime}} OR (logtime:${end.logtime} AND id:[* TO ${end.id}])
-infra-manager.jobs.solr_data_export.archive_service_logs.read_block_size=2000
-infra-manager.jobs.solr_data_export.archive_service_logs.write_block_size=1000
-infra-manager.jobs.solr_data_export.archive_service_logs.destination=HDFS
-infra-manager.jobs.solr_data_export.archive_service_logs.file_name_suffix_column=logtime
-infra-manager.jobs.solr_data_export.archive_service_logs.file_name_suffix_date_format=yyyy-MM-dd'T'HH-mm-ss.SSSX
-infra-manager.jobs.solr_data_export.archive_service_logs.hdfs_endpoint=hdfs://c6401.ambari.apache.org:8020
-infra-manager.jobs.solr_data_export.archive_service_logs.hdfs_destination_directory=/archived_service_logs
+infra-manager.jobs.solr_data_archiving.archive_service_logs.enabled=true
+infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.zoo_keeper_connection_string=c6401.ambari.apache.org:2181/infra-solr
+infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.collection=hadoop_logs
+infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.query_text=logtime:[${start} TO ${end}]
+infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.filter_query_text=(logtime:${logtime} AND id:{${id} TO *]) OR logtime:{${logtime} TO ${end}]
+infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.sort_column[0]=logtime
+infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.sort_column[1]=id
+infra-manager.jobs.solr_data_archiving.archive_service_logs.solr.delete_query_text=logtime:[${start.logtime} TO ${end.logtime}} OR (logtime:${end.logtime} AND id:[* TO ${end.id}])
+infra-manager.jobs.solr_data_archiving.archive_service_logs.read_block_size=2000
+infra-manager.jobs.solr_data_archiving.archive_service_logs.write_block_size=1000
+infra-manager.jobs.solr_data_archiving.archive_service_logs.destination=HDFS
+infra-manager.jobs.solr_data_archiving.archive_service_logs.file_name_suffix_column=logtime
+infra-manager.jobs.solr_data_archiving.archive_service_logs.file_name_suffix_date_format=yyyy-MM-dd'T'HH-mm-ss.SSSX
+infra-manager.jobs.solr_data_archiving.archive_service_logs.hdfs_endpoint=hdfs://c6401.ambari.apache.org:8020
+infra-manager.jobs.solr_data_archiving.archive_service_logs.hdfs_destination_directory=/archived_service_logs
# Note: set hdfs user using the HADOOP_USER_NAME environmental variable. Value: hdfs
\ No newline at end of file
diff --git a/ambari-infra/ambari-infra-solr-client/pom.xml b/ambari-infra/ambari-infra-solr-client/pom.xml
index 0d32374..8782a97 100644
--- a/ambari-infra/ambari-infra-solr-client/pom.xml
+++ b/ambari-infra/ambari-infra-solr-client/pom.xml
@@ -53,15 +53,16 @@
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
- <version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
+ <version>1.9.13</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
+ <version>1.8</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
@@ -80,6 +81,29 @@
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
+ <version>1.2.17</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.mail</groupId>
+ <artifactId>mail</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jmx</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jms</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
diff --git a/ambari-infra/ambari-infra-solr-client/src/main/resources/solrCloudCli.sh b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrCloudCli.sh
index ac7d59f..69f9b2c 100644
--- a/ambari-infra/ambari-infra-solr-client/src/main/resources/solrCloudCli.sh
+++ b/ambari-infra/ambari-infra-solr-client/src/main/resources/solrCloudCli.sh
@@ -23,4 +23,4 @@ if [ "$sdir" != "$ldir" ]; then
DIR="$ldir"
fi
-PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR:$DIR/libs/*" org.apache.ambari.infra.solr.AmbariSolrCloudCLI ${1+"$@"}
\ No newline at end of file
+PATH=$JAVA_HOME/bin:$PATH $JVM -classpath "$DIR:$DIR/libs/*" $INFRA_SOLR_CLI_OPTS org.apache.ambari.infra.solr.AmbariSolrCloudCLI ${1+"$@"}
\ No newline at end of file
diff --git a/ambari-infra/pom.xml b/ambari-infra/pom.xml
index 2400493..aab1727 100644
--- a/ambari-infra/pom.xml
+++ b/ambari-infra/pom.xml
@@ -17,13 +17,8 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <groupId>org.apache.ambari</groupId>
- <artifactId>ambari-project</artifactId>
- <version>2.0.0.0-SNAPSHOT</version>
- <relativePath>../ambari-project</relativePath>
- </parent>
<modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.ambari</groupId>
<artifactId>ambari-infra</artifactId>
<version>2.0.0.0-SNAPSHOT</version>
<packaging>pom</packaging>
@@ -37,8 +32,40 @@
<deb.architecture>amd64</deb.architecture>
<deb.dependency.list>${deb.python.ver}</deb.dependency.list>
<hadoop.version>3.0.0</hadoop.version>
+ <surefire.argLine>-Xmx1024m -Xms512m</surefire.argLine>
+ <skipSurefireTests>false</skipSurefireTests>
</properties>
+ <repositories>
+ <repository>
+ <id>oss.sonatype.org</id>
+ <name>OSS Sonatype Staging</name>
+ <url>https://oss.sonatype.org/content/groups/staging</url>
+ </repository>
+ <repository>
+ <id>spring-milestones</id>
+ <name>Spring Milestones</name>
+ <url>http://repo.spring.io/milestone</url>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>ASF Staging</id>
+ <url>https://repository.apache.org/content/groups/staging/</url>
+ </repository>
+ <repository>
+ <id>ASF Snapshots</id>
+ <url>https://repository.apache.org/content/repositories/snapshots/</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ <releases>
+ <enabled>false</enabled>
+ </releases>
+ </repository>
+ </repositories>
+
<modules>
<module>ambari-infra-assembly</module>
<module>ambari-infra-solr-client</module>
@@ -48,6 +75,30 @@
</modules>
<build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>1.4.1</version>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>properties-maven-plugin</artifactId>
+ <version>1.0.0</version>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2.1</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>2.20</version>
+ </plugin>
+ </plugins>
+ </pluginManagement>
<plugins>
<plugin>
<inherited>false</inherited>
@@ -70,9 +121,9 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
+ <version>2.20</version>
<configuration>
<skip>${skipSurefireTests}</skip>
- <groups>${testcase.groups}</groups>
</configuration>
</plugin>
<plugin>
@@ -86,6 +137,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-clean-plugin</artifactId>
+ <version>2.5</version>
<configuration>
<filesets>
<fileset>
@@ -156,6 +208,7 @@
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
+ <version>0.11</version>
<configuration>
<excludes>
<exclude>README.md</exclude>
@@ -179,14 +232,29 @@
</plugins>
</build>
- <dependencies>
- <!-- Dependency in order to annotate unit tests with a category. -->
- <dependency>
- <groupId>org.apache.ambari</groupId>
- <artifactId>ambari-utility</artifactId>
- <version>1.0.0.0-SNAPSHOT</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.8</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.5</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
</project>
diff --git a/ambari-logsearch/ambari-logsearch-appender/pom.xml b/ambari-logsearch/ambari-logsearch-appender/pom.xml
index 1580737..65a68a9 100644
--- a/ambari-logsearch/ambari-logsearch-appender/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-appender/pom.xml
@@ -66,6 +66,29 @@
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
+ <version>1.2.17</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.mail</groupId>
+ <artifactId>mail</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jmx</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jms</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
index c116378..a70b713 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
@@ -61,6 +61,7 @@
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
+ <version>1.8</version>
</dependency>
<dependency>
<groupId>junit</groupId>
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
index bebf225..47135b6 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
@@ -19,6 +19,7 @@
package org.apache.ambari.logfeeder.output;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.hash.Hashing;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.loglevelfilter.LogLevelFilterHandler;
@@ -29,7 +30,6 @@ import org.apache.ambari.logfeeder.plugin.input.InputMarker;
import org.apache.ambari.logfeeder.plugin.manager.OutputManager;
import org.apache.ambari.logfeeder.plugin.output.Output;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.ambari.logfeeder.util.MurmurHash;
import org.apache.ambari.logsearch.config.api.OutputConfigMonitor;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Level;
@@ -46,7 +46,6 @@ import java.util.UUID;
public class OutputManagerImpl extends OutputManager {
private static final Logger LOG = Logger.getLogger(OutputManagerImpl.class);
- private static final int HASH_SEED = 31174077;
private static final int MAX_OUTPUT_SIZE = 32765; // 32766-1
private List<Output> outputs = new ArrayList<Output>();
@@ -132,7 +131,9 @@ public class OutputManagerImpl extends OutputManager {
}
}
- Long eventMD5 = MurmurHash.hash64A(LogFeederUtil.getGson().toJson(jsonObj).getBytes(), HASH_SEED);
+
+ byte[] bytes = LogFeederUtil.getGson().toJson(jsonObj).getBytes();
+ Long eventMD5 = Hashing.md5().hashBytes(bytes).asLong();
if (input.isGenEventMD5()) {
jsonObj.put("event_md5", prefix + eventMD5.toString());
}
@@ -157,7 +158,7 @@ public class OutputManagerImpl extends OutputManager {
String logMessage = (String) jsonObj.get("log_message");
logMessage = truncateLongLogMessage(jsonObj, input, logMessage);
if (addMessageMD5) {
- jsonObj.put("message_md5", "" + MurmurHash.hash64A(logMessage.getBytes(), 31174077));
+ jsonObj.put("message_md5", "" + Hashing.md5().hashBytes(logMessage.getBytes()).asLong());
}
}
if (logLevelFilterHandler.isAllowed(jsonObj, inputMarker)
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/MurmurHash.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/MurmurHash.java
deleted file mode 100644
index dbbefaf..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/MurmurHash.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.logfeeder.util;
-
-import com.google.common.primitives.Ints;
-
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-
-/**
- * This is a very fast, non-cryptographic hash suitable for general hash-based
- * lookup. See http://murmurhash.googlepages.com/ for more details.
- * <p/>
- * <p>The C version of MurmurHash 2.0 found at that site was ported
- * to Java by Andrzej Bialecki (ab at getopt org).</p>
- */
-public final class MurmurHash {
-
- private MurmurHash() {
- }
-
- /**
- * Hashes an int.
- *
- * @param data The int to hash.
- * @param seed The seed for the hash.
- * @return The 32 bit hash of the bytes in question.
- */
- public static int hash(int data, int seed) {
- return hash(ByteBuffer.wrap(Ints.toByteArray(data)), seed);
- }
-
- /**
- * Hashes bytes in an array.
- *
- * @param data The bytes to hash.
- * @param seed The seed for the hash.
- * @return The 32 bit hash of the bytes in question.
- */
- public static int hash(byte[] data, int seed) {
- return hash(ByteBuffer.wrap(data), seed);
- }
-
- /**
- * Hashes bytes in part of an array.
- *
- * @param data The data to hash.
- * @param offset Where to start munging.
- * @param length How many bytes to process.
- * @param seed The seed to start with.
- * @return The 32-bit hash of the data in question.
- */
- public static int hash(byte[] data, int offset, int length, int seed) {
- return hash(ByteBuffer.wrap(data, offset, length), seed);
- }
-
- /**
- * Hashes the bytes in a buffer from the current position to the limit.
- *
- * @param buf The bytes to hash.
- * @param seed The seed for the hash.
- * @return The 32 bit murmur hash of the bytes in the buffer.
- */
- public static int hash(ByteBuffer buf, int seed) {
- // save byte order for later restoration
- ByteOrder byteOrder = buf.order();
- buf.order(ByteOrder.LITTLE_ENDIAN);
-
- int m = 0x5bd1e995;
- int r = 24;
-
- int h = seed ^ buf.remaining();
-
- while (buf.remaining() >= 4) {
- int k = buf.getInt();
-
- k *= m;
- k ^= k >>> r;
- k *= m;
-
- h *= m;
- h ^= k;
- }
-
- if (buf.remaining() > 0) {
- ByteBuffer finish = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN);
- // for big-endian version, use this first:
- // finish.position(4-buf.remaining());
- finish.put(buf).rewind();
- h ^= finish.getInt();
- h *= m;
- }
-
- h ^= h >>> 13;
- h *= m;
- h ^= h >>> 15;
-
- buf.order(byteOrder);
- return h;
- }
-
-
- public static long hash64A(byte[] data, int seed) {
- return hash64A(ByteBuffer.wrap(data), seed);
- }
-
- public static long hash64A(byte[] data, int offset, int length, int seed) {
- return hash64A(ByteBuffer.wrap(data, offset, length), seed);
- }
-
- public static long hash64A(ByteBuffer buf, int seed) {
- ByteOrder byteOrder = buf.order();
- buf.order(ByteOrder.LITTLE_ENDIAN);
-
- long m = 0xc6a4a7935bd1e995L;
- int r = 47;
-
- long h = seed ^ (buf.remaining() * m);
-
- while (buf.remaining() >= 8) {
- long k = buf.getLong();
-
- k *= m;
- k ^= k >>> r;
- k *= m;
-
- h ^= k;
- h *= m;
- }
-
- if (buf.remaining() > 0) {
- ByteBuffer finish = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN);
- // for big-endian version, do this first:
- // finish.position(8-buf.remaining());
- finish.put(buf).rewind();
- h ^= finish.getLong();
- h *= m;
- }
-
- h ^= h >>> r;
- h *= m;
- h ^= h >>> r;
-
- buf.order(byteOrder);
- return h;
- }
-
-}
diff --git a/ambari-logsearch/ambari-logsearch-server/pom.xml b/ambari-logsearch/ambari-logsearch-server/pom.xml
index 6a7debf..02c6ab2 100755
--- a/ambari-logsearch/ambari-logsearch-server/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-server/pom.xml
@@ -434,10 +434,12 @@
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
+ <version>1.8</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
+ <version>2.5</version>
</dependency>
<dependency>
<groupId>cglib</groupId>
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/auth/filter/AbstractJWTFilter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/auth/filter/AbstractJWTFilter.java
index 80fbf6b..2edcf88 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/auth/filter/AbstractJWTFilter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/auth/filter/AbstractJWTFilter.java
@@ -18,6 +18,7 @@
*/
package org.apache.ambari.logsearch.auth.filter;
+import com.google.gson.Gson;
import io.jsonwebtoken.Claims;
import io.jsonwebtoken.ExpiredJwtException;
import io.jsonwebtoken.Jwts;
@@ -46,12 +47,15 @@ import javax.servlet.http.HttpServletResponse;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.security.interfaces.RSAPublicKey;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
public abstract class AbstractJWTFilter extends AbstractAuthenticationProcessingFilter {
@@ -110,14 +114,27 @@ public abstract class AbstractJWTFilter extends AbstractAuthenticationProcessing
@Override
protected void successfulAuthentication(HttpServletRequest request, HttpServletResponse response, FilterChain chain, Authentication authResult) throws IOException, ServletException {
super.successfulAuthentication(request, response, chain, authResult);
- response.sendRedirect(request.getRequestURL().toString() + getOriginalQueryString(request));
+ String ajaxRequestHeader = request.getHeader("X-Requested-With");
+ if (isWebUserAgent(request.getHeader("User-Agent")) && !"XMLHttpRequest".equals(ajaxRequestHeader)) {
+ response.sendRedirect(request.getRequestURL().toString() + getOriginalQueryString(request));
+ }
+ // chain.doFilter(request, response); TODO: check
}
@Override
protected void unsuccessfulAuthentication(HttpServletRequest request, HttpServletResponse response, AuthenticationException failed) throws IOException, ServletException {
super.unsuccessfulAuthentication(request, response, failed);
+ String ajaxRequestHeader = request.getHeader("X-Requested-With");
String loginUrl = constructLoginURL(request);
- response.sendRedirect(loginUrl);
+ if (!isWebUserAgent(request.getHeader("User-Agent")) || "XMLHttpRequest".equals(ajaxRequestHeader)) {
+ Map<String, String> mapObj = new HashMap<>();
+ mapObj.put("knoxssoredirectURL", URLEncoder.encode(loginUrl, "UTF-8"));
+ response.setContentType("application/json");
+ response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+ response.sendError(HttpServletResponse.SC_UNAUTHORIZED, new Gson().toJson(mapObj));
+ } else {
+ response.sendRedirect(loginUrl);
+ }
}
private String getJWTFromCookie(HttpServletRequest req) {
@@ -135,6 +152,20 @@ public abstract class AbstractJWTFilter extends AbstractAuthenticationProcessing
return serializedJWT;
}
+ private boolean isWebUserAgent(String userAgent) {
+ boolean isWeb = false;
+ List<String> userAgentList = getUserAgentList();
+ if (userAgentList != null && userAgentList.size() > 0) {
+ for (String ua : userAgentList) {
+ if (StringUtils.startsWithIgnoreCase(userAgent, ua)) {
+ isWeb = true;
+ break;
+ }
+ }
+ }
+ return isWeb;
+ }
+
private RSAPublicKey parseRSAPublicKey(String pem) throws ServletException {
String fullPem = PEM_HEADER + pem + PEM_FOOTER;
try {
@@ -190,4 +221,6 @@ public abstract class AbstractJWTFilter extends AbstractAuthenticationProcessing
protected abstract Collection<? extends GrantedAuthority> getAuthorities();
+ protected abstract List<String> getUserAgentList();
+
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
index c316346..4722b79 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java
@@ -33,7 +33,7 @@ public class LogSearchConstants {
public static final String TRACE = "TRACE";
public static final String FATAL = "FATAL";
public static final String UNKNOWN = "UNKNOWN";
-
+
public static final String[] SUPPORTED_LOG_LEVELS = {FATAL, ERROR, WARN, INFO, DEBUG, TRACE, UNKNOWN};
// Application Constants
@@ -60,7 +60,11 @@ public class LogSearchConstants {
public static final String LOGFEEDER_FILTER_NAME = "log_feeder_config";
public static final String SORT = "sort";
-
+
+ // info features constants
+ public static final String SHIPPER_CONFIG_API_KEY = "metadata_patterns";
+ public static final String AUTH_FEATURE_KEY = "auth";
+
//Facet Constant
public static final String FACET_FIELD = "facet.field";
public static final String FACET_PIVOT = "facet.pivot";
@@ -108,6 +112,10 @@ public class LogSearchConstants {
public static final String REQUEST_PARAM_ROW_TYPE = "rowType";
public static final String REQUEST_PARAM_UTC_OFFSET = "utcOffset";
public static final String REQUEST_PARAM_HOSTS = "hostList";
-
-
+ public static final String REQUEST_PARAM_USERS = "userList";
+ public static final String REQUEST_PARAM_PAGE_DEFAULT_VALUE = "0";
+ public static final String REQUEST_PARAM_PAGE_SIZE_DEFAULT_VALUE = "1000";
+ public static final String REQUEST_PARAM_SHIPPER_CONFIG = "shipperConfig";
+ public static final String REQUEST_PARAM_LOG_ID = "logId";
+ public static final String REQUEST_PARAM_TEST_ENTRY = "testEntry";
}
\ No newline at end of file
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java
index 9dd8b34..228f1c6 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/MessageEnums.java
@@ -34,7 +34,7 @@ public enum MessageEnums {
ZK_CONFIG_NOT_READY("logsearch.zk.config.error", "Collection configuration has not uploaded yet"),
SOLR_COLLECTION_NOT_READY("logsearch.solr.collection.error", "Solr has not accessible yet for collection."),
CONFIGURATION_NOT_AVAILABLE("logsearch.config.not_available", "Log Search configuration is not available"),
-
+ CONFIGURATION_API_DISABLED("logsearch.config.api.disabled", "Log Search configuration is not available"),
// Common Validations
INVALID_PASSWORD("logsearch.validation.invalid_password", "Invalid password"),
INVALID_INPUT_DATA("logsearch.validation.invalid_input_data", "Invalid input data"),
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/AuthPropsConfig.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/AuthPropsConfig.java
index 06673b3..2facf86 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/AuthPropsConfig.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/AuthPropsConfig.java
@@ -137,7 +137,7 @@ public class AuthPropsConfig {
sources = {LOGSEARCH_PROPERTIES_FILE}
)
private String cookieName;
- @Value("${logsearch.auth.jwt.query.param.original_url:originalUrl=}")
+ @Value("${logsearch.auth.jwt.query.param.original_url:originalUrl}")
@LogSearchPropertyDescription(
name = "logsearch.auth.jwt.query.param.original_url",
description = "Name of the original request URL which is used to redirect to Log Search Portal.",
@@ -157,6 +157,16 @@ public class AuthPropsConfig {
)
private List<String> audiences;
+ @Value("#{'${logsearch.auth.jwt.user.agents:Mozilla,Opera,Chrome}'.split(',')}")
+ @LogSearchPropertyDescription(
+ name = "logsearch.auth.jwt.user.agents",
+ description = "Comma separated web user agent list. (Used as prefixes)",
+ examples = {"Mozilla,Chrome"},
+ defaultValue = "Mozilla,Opera,Chrome",
+ sources = {LOGSEARCH_PROPERTIES_FILE}
+ )
+ private List<String> userAgentList;
+
@Value("#{'${logsearch.roles.allowed:AMBARI.ADMINISTRATOR,CLUSTER.ADMINISTRATOR}'.split(',')}")
@LogSearchPropertyDescription(
name = "logsearch.roles.allowed",
@@ -296,4 +306,12 @@ public class AuthPropsConfig {
public void setRedirectForward(boolean redirectForward) {
this.redirectForward = redirectForward;
}
+
+ public List<String> getUserAgentList() {
+ return this.userAgentList;
+ }
+
+ public void setUserAgentList(List<String> userAgentList) {
+ this.userAgentList = userAgentList;
+ }
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingPropertyMap.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/LogSearchConfigApiConfig.java
similarity index 51%
copy from ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingPropertyMap.java
copy to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/LogSearchConfigApiConfig.java
index fccfd59..6e16179 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/deleting/DocumentDeletingPropertyMap.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/LogSearchConfigApiConfig.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -16,29 +16,32 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.infra.job.deleting;
+package org.apache.ambari.logsearch.conf;
-import org.apache.ambari.infra.job.PropertyMap;
-import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
+import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
-import java.util.Map;
+import static org.apache.ambari.logsearch.common.LogSearchConstants.LOGSEARCH_PROPERTIES_FILE;
@Configuration
-@ConfigurationProperties(prefix = "infra-manager.jobs")
-public class DocumentDeletingPropertyMap implements PropertyMap<DocumentDeletingProperties> {
- private Map<String, DocumentDeletingProperties> solrDataDeleting;
+public class LogSearchConfigApiConfig {
- public Map<String, DocumentDeletingProperties> getSolrDataDeleting() {
- return solrDataDeleting;
- }
+ @Value("${logsearch.config.api.enabled:true}")
+ @LogSearchPropertyDescription(
+ name = "logsearch.config.api.enabled",
+ description = "Enable config API feature and shipperconfig API endpoints.",
+ examples = {"false"},
+ defaultValue = "true",
+ sources = {LOGSEARCH_PROPERTIES_FILE}
+ )
+ private boolean configApiEnabled;
- public void setSolrDataDeleting(Map<String, DocumentDeletingProperties> solrDataDeleting) {
- this.solrDataDeleting = solrDataDeleting;
+ public boolean isConfigApiEnabled() {
+ return configApiEnabled;
}
- @Override
- public Map<String, DocumentDeletingProperties> getPropertyMap() {
- return getSolrDataDeleting();
+ public void setConfigApiEnabled(boolean configApiEnabled) {
+ this.configApiEnabled = configApiEnabled;
}
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/LogSearchSslConfig.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/LogSearchSslConfig.java
index afa2293..15579b6 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/LogSearchSslConfig.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/LogSearchSslConfig.java
@@ -27,7 +27,7 @@ import static org.apache.ambari.logsearch.common.LogSearchConstants.LOGSEARCH_PR
@Configuration
public class LogSearchSslConfig {
- public static final String LOGSEARCH_CERT_DEFAULT_FOLDER = "/etc/ambari-logsearch-portal/conf/keys";
+ public static final String LOGSEARCH_CERT_DEFAULT_FOLDER = "/usr/lib/ambari-logsearch-portal/conf/keys";
public static final String LOGSEARCH_CERT_DEFAULT_ALGORITHM = "sha256WithRSA";
public static final String CREDENTIAL_STORE_PROVIDER_PATH = "hadoop.security.credential.provider.path";
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java
index 6f8d7ba..338fc21 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java
@@ -22,6 +22,7 @@ import com.google.common.collect.Lists;
import org.apache.ambari.logsearch.conf.global.LogSearchConfigState;
import org.apache.ambari.logsearch.conf.global.SolrCollectionState;
+import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
import org.apache.ambari.logsearch.web.authenticate.LogsearchAuthFailureHandler;
import org.apache.ambari.logsearch.web.authenticate.LogsearchAuthSuccessHandler;
import org.apache.ambari.logsearch.web.authenticate.LogsearchLogoutSuccessHandler;
@@ -43,7 +44,6 @@ import org.springframework.security.config.annotation.web.configuration.EnableWe
import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
import org.springframework.security.config.http.SessionCreationPolicy;
import org.springframework.security.web.access.intercept.FilterSecurityInterceptor;
-import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
import org.springframework.security.web.authentication.www.BasicAuthenticationFilter;
import org.springframework.security.web.util.matcher.AntPathRequestMatcher;
import org.springframework.security.web.util.matcher.OrRequestMatcher;
@@ -51,6 +51,7 @@ import org.springframework.security.web.util.matcher.RequestMatcher;
import javax.inject.Inject;
import javax.inject.Named;
+import java.util.ArrayList;
import java.util.List;
import static org.apache.ambari.logsearch.common.LogSearchConstants.LOGSEARCH_SESSION_ID;
@@ -89,6 +90,9 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
@Inject
private LogSearchConfigState logSearchConfigState;
+ @Inject
+ private LogSearchConfigApiConfig logSearchConfigApiConfig;
+
@Override
protected void configure(HttpSecurity http) throws Exception {
http
@@ -117,7 +121,7 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
.addFilterBefore(logsearchJwtFilter(), LogsearchSecurityContextFormationFilter.class)
.logout()
.logoutUrl("/logout")
- .deleteCookies(LOGSEARCH_SESSION_ID)
+ .deleteCookies(getCookies())
.logoutSuccessHandler(new LogsearchLogoutSuccessHandler());
}
@@ -184,7 +188,7 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
@Bean
public LogSearchConfigStateFilter logSearchConfigStateFilter() {
- return new LogSearchConfigStateFilter(logsearchConfigRequestMatcher(), logSearchConfigState);
+ return new LogSearchConfigStateFilter(logsearchConfigRequestMatcher(), logSearchConfigState, logSearchConfigApiConfig.isConfigApiEnabled());
}
@Bean
@@ -193,7 +197,9 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
matchers.add(new AntPathRequestMatcher("/docs/**"));
matchers.add(new AntPathRequestMatcher("/swagger-ui/**"));
matchers.add(new AntPathRequestMatcher("/swagger.html"));
- matchers.add(new AntPathRequestMatcher("/"));
+ if (!authPropsConfig.isAuthJwtEnabled()) {
+ matchers.add(new AntPathRequestMatcher("/"));
+ }
matchers.add(new AntPathRequestMatcher("/login"));
matchers.add(new AntPathRequestMatcher("/logout"));
matchers.add(new AntPathRequestMatcher("/resources/**"));
@@ -202,7 +208,6 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
matchers.add(new AntPathRequestMatcher("/assets/**"));
matchers.add(new AntPathRequestMatcher("/templates/**"));
matchers.add(new AntPathRequestMatcher("/api/v1/info/**"));
- matchers.add(new AntPathRequestMatcher("/api/v1/public/**"));
matchers.add(new AntPathRequestMatcher("/api/v1/swagger.json"));
matchers.add(new AntPathRequestMatcher("/api/v1/swagger.yaml"));
return new OrRequestMatcher(matchers);
@@ -224,4 +229,13 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter {
return new AntPathRequestMatcher("/api/v1/shipper/**");
}
+ private String[] getCookies() {
+ List<String> cookies = new ArrayList<>();
+ cookies.add(LOGSEARCH_SESSION_ID);
+ if (authPropsConfig.isAuthJwtEnabled()) {
+ cookies.add(authPropsConfig.getCookieName());
+ }
+ return cookies.toArray(new String[0]);
+ }
+
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
index baf5adb..fc71409 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/configurer/LogSearchConfigConfigurer.java
@@ -23,6 +23,7 @@ import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.inject.Named;
+import org.apache.ambari.logsearch.conf.LogSearchConfigApiConfig;
import org.apache.ambari.logsearch.conf.LogSearchConfigMapHolder;
import org.apache.ambari.logsearch.conf.global.LogSearchConfigState;
import org.apache.ambari.logsearch.config.api.LogSearchConfigFactory;
@@ -48,6 +49,9 @@ public class LogSearchConfigConfigurer implements Configurer {
@Inject
private LogSearchConfigMapHolder logSearchConfigMapHolder;
+ @Inject
+ private LogSearchConfigApiConfig logSearchConfigApiConfig;
+
@PostConstruct
@Override
public void start() {
@@ -57,9 +61,13 @@ public class LogSearchConfigConfigurer implements Configurer {
logger.info("Started thread to set up log search config");
while (true) {
try {
- logSearchConfig = LogSearchConfigFactory.createLogSearchConfigServer(logSearchConfigMapHolder.getLogsearchProperties(),
+ if (logSearchConfigApiConfig.isConfigApiEnabled()) {
+ logSearchConfig = LogSearchConfigFactory.createLogSearchConfigServer(logSearchConfigMapHolder.getLogsearchProperties(),
LogSearchConfigServerZK.class);
- logSearchConfigState.setLogSearchConfigAvailable(true);
+ logSearchConfigState.setLogSearchConfigAvailable(true);
+ } else {
+ logger.info("Config API is disabled. Shipper configs won't be accessible from the Rest API.");
+ }
break;
} catch (Exception e) {
logger.warn("Could not initialize Log Search config, going to sleep for " + RETRY_INTERVAL_SECONDS + " seconds ", e);
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
index c92dd14..7ea5fbb 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java
@@ -85,6 +85,18 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
return query;
}
+ public SolrQuery addInFiltersIfNotNullAndEnabled(SolrQuery query, String value, String field, boolean condition) {
+ if (condition) {
+ List<String> valuesList = value.length() == 0 ? Arrays.asList("\\-1") : splitValueAsList(value, ",");
+ if (valuesList.size() > 1) {
+ query.addFilterQuery(String.format("%s:(%s)", field, StringUtils.join(valuesList, " OR ")));
+ } else {
+ query.addFilterQuery(String.format("%s:%s", field, valuesList.get(0)));
+ }
+ }
+ return query;
+ }
+
public Query addInFilterQuery(Query query, String field, List<String> values, boolean negate) {
if (CollectionUtils.isNotEmpty(values)) {
addFilterQuery(query, new Criteria(field).is(values), negate);
@@ -176,5 +188,4 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
String fieldTypeMetaData = schemaFieldTypeMap.get(fieldType);
return SolrUtil.putWildCardByType(fieldEntry.getValue(), fieldType, fieldTypeMetaData);
}
-
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java
index 06dfae9..beb8326 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java
@@ -20,10 +20,15 @@ package org.apache.ambari.logsearch.converter;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.model.request.impl.AuditBarGraphRequest;
+import org.apache.ambari.logsearch.solr.SolrConstants;
+import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrQuery;
import javax.inject.Named;
+import java.util.Arrays;
+import java.util.List;
+
import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_COMPONENT;
import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_EVTTIME;
import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.CLUSTER;
@@ -50,6 +55,9 @@ public class AuditBarGraphRequestQueryConverter extends AbstractDateRangeFacetQu
public SolrQuery convert(AuditBarGraphRequest request) {
SolrQuery query = super.convert(request);
addListFilterToSolrQuery(query, CLUSTER, request.getClusters());
+ addInFiltersIfNotNullAndEnabled(query, request.getUserList(),
+ SolrConstants.AuditLogConstants.AUDIT_REQUEST_USER,
+ StringUtils.isNotBlank(request.getUserList()));
return query;
}
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditComponentsRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditComponentsRequestQueryConverter.java
index bf77444..55280a8 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditComponentsRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditComponentsRequestQueryConverter.java
@@ -20,7 +20,11 @@ package org.apache.ambari.logsearch.converter;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.model.request.impl.AuditComponentRequest;
+import org.apache.ambari.logsearch.model.request.impl.AuditServiceLoadRequest;
+import org.apache.ambari.logsearch.solr.SolrConstants;
+import org.apache.commons.lang.StringUtils;
import org.springframework.data.solr.core.query.FacetOptions;
+import org.springframework.data.solr.core.query.SimpleFacetQuery;
import javax.inject.Named;
@@ -50,4 +54,11 @@ public class AuditComponentsRequestQueryConverter extends AbstractLogRequestFace
facetOptions.addFacetOnField(AUDIT_COMPONENT);
facetOptions.setFacetLimit(-1);
}
+
+ @Override
+ public void appendFacetQuery(SimpleFacetQuery facetQuery, AuditComponentRequest request) {
+ addInFiltersIfNotNullAndEnabled(facetQuery, request.getUserList(),
+ SolrConstants.AuditLogConstants.AUDIT_REQUEST_USER,
+ StringUtils.isNotBlank(request.getUserList()));
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditLogRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditLogRequestQueryConverter.java
index a2c1d81..eba91eb 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditLogRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditLogRequestQueryConverter.java
@@ -20,6 +20,9 @@ package org.apache.ambari.logsearch.converter;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.model.request.impl.AuditLogRequest;
+import org.apache.ambari.logsearch.solr.SolrConstants;
+import org.apache.commons.lang.StringUtils;
+import org.apache.solr.client.solrj.SolrQuery;
import org.springframework.data.solr.core.query.SimpleQuery;
import javax.inject.Named;
@@ -29,6 +32,10 @@ public class AuditLogRequestQueryConverter extends AbstractAuditLogRequestQueryC
@Override
public SimpleQuery extendLogQuery(AuditLogRequest request, SimpleQuery query) {
+ addInFiltersIfNotNullAndEnabled(
+ query, request.getUserList(),
+ SolrConstants.AuditLogConstants.AUDIT_REQUEST_USER,
+ StringUtils.isNotBlank(request.getUserList()));
return query;
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverter.java
index d88ec6d..26a8d26 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverter.java
@@ -20,7 +20,10 @@ package org.apache.ambari.logsearch.converter;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.model.request.impl.AuditServiceLoadRequest;
+import org.apache.ambari.logsearch.solr.SolrConstants;
+import org.apache.commons.lang.StringUtils;
import org.springframework.data.solr.core.query.FacetOptions;
+import org.springframework.data.solr.core.query.SimpleFacetQuery;
import javax.inject.Named;
@@ -50,4 +53,11 @@ public class AuditServiceLoadRequestQueryConverter extends AbstractLogRequestFac
facetOptions.addFacetOnField(AUDIT_COMPONENT);
facetOptions.setFacetLimit(10);
}
+
+ @Override
+ public void appendFacetQuery(SimpleFacetQuery facetQuery, AuditServiceLoadRequest request) {
+ addInFiltersIfNotNullAndEnabled(facetQuery, request.getUserList(),
+ SolrConstants.AuditLogConstants.AUDIT_REQUEST_USER,
+ StringUtils.isNotBlank(request.getUserList()));
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverter.java
index bf6a457..c4b8ae0 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverter.java
@@ -20,7 +20,10 @@ package org.apache.ambari.logsearch.converter;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.model.request.impl.FieldAuditLogRequest;
+import org.apache.ambari.logsearch.solr.SolrConstants;
+import org.apache.commons.lang.StringUtils;
import org.springframework.data.solr.core.query.FacetOptions;
+import org.springframework.data.solr.core.query.SimpleFacetQuery;
import javax.inject.Named;
@@ -50,4 +53,10 @@ public class FieldAuditLogRequestQueryConverter extends AbstractLogRequestFacetQ
return LogType.AUDIT;
}
+ @Override
+ public void appendFacetQuery(SimpleFacetQuery facetQuery, FieldAuditLogRequest request) {
+ addInFiltersIfNotNullAndEnabled(facetQuery, request.getUserList(),
+ SolrConstants.AuditLogConstants.AUDIT_REQUEST_USER,
+ StringUtils.isNotBlank(request.getUserList()));
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/TopFieldAuditLogRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/TopFieldAuditLogRequestQueryConverter.java
index 9985a18..636c7e9 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/TopFieldAuditLogRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/TopFieldAuditLogRequestQueryConverter.java
@@ -20,7 +20,10 @@ package org.apache.ambari.logsearch.converter;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.model.request.impl.TopFieldAuditLogRequest;
+import org.apache.ambari.logsearch.solr.SolrConstants;
+import org.apache.commons.lang.StringUtils;
import org.springframework.data.solr.core.query.FacetOptions;
+import org.springframework.data.solr.core.query.SimpleFacetQuery;
import javax.inject.Named;
@@ -51,4 +54,10 @@ public class TopFieldAuditLogRequestQueryConverter extends AbstractLogRequestFac
return LogType.AUDIT;
}
+ @Override
+ public void appendFacetQuery(SimpleFacetQuery facetQuery, TopFieldAuditLogRequest request) {
+ addInFiltersIfNotNullAndEnabled(facetQuery, request.getUserList(),
+ SolrConstants.AuditLogConstants.AUDIT_REQUEST_USER,
+ StringUtils.isNotBlank(request.getUserList()));
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverter.java
index 6b7c00e..2a42310 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverter.java
@@ -20,7 +20,10 @@ package org.apache.ambari.logsearch.converter;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.model.request.impl.UserExportRequest;
+import org.apache.ambari.logsearch.solr.SolrConstants;
+import org.apache.commons.lang.StringUtils;
import org.springframework.data.solr.core.query.FacetOptions;
+import org.springframework.data.solr.core.query.SimpleFacetQuery;
import javax.inject.Named;
@@ -52,4 +55,11 @@ public class UserExportRequestQueryConverter extends AbstractLogRequestFacetQuer
public LogType getLogType() {
return LogType.AUDIT;
}
+
+ @Override
+ public void appendFacetQuery(SimpleFacetQuery facetQuery, UserExportRequest request) {
+ addInFiltersIfNotNullAndEnabled(facetQuery, request.getUserList(),
+ SolrConstants.AuditLogConstants.AUDIT_REQUEST_USER,
+ StringUtils.isNotBlank(request.getUserList()));
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index 05217d9..218d1a0 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -22,6 +22,7 @@ package org.apache.ambari.logsearch.dao;
import org.apache.ambari.logsearch.common.LogSearchContext;
import org.apache.ambari.logsearch.common.LogType;
import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.conf.LogSearchConfigApiConfig;
import org.apache.ambari.logsearch.conf.SolrKerberosConfig;
import org.apache.ambari.logsearch.conf.SolrPropsConfig;
import org.apache.ambari.logsearch.conf.global.LogSearchConfigState;
@@ -61,6 +62,9 @@ public abstract class SolrDaoBase {
private LogSearchConfigState logSearchConfigState;
@Inject
+ private LogSearchConfigApiConfig logSearchConfigApiConfig;
+
+ @Inject
private LogSearchConfigConfigurer logSearchConfigConfigurer;
protected SolrDaoBase(LogType logType) {
@@ -68,9 +72,15 @@ public abstract class SolrDaoBase {
}
public void waitForLogSearchConfig() {
- while (!logSearchConfigState.isLogSearchConfigAvailable()) {
- LOG.info("Log Search config not available yet, waiting...");
- try { Thread.sleep(1000); } catch (Exception e) { LOG.warn("Exception during waiting for Log Search Config", e); }
+ if (logSearchConfigApiConfig.isConfigApiEnabled()) {
+ while (!logSearchConfigState.isLogSearchConfigAvailable()) {
+ LOG.info("Log Search config not available yet, waiting...");
+ try {
+ Thread.sleep(1000);
+ } catch (Exception e) {
+ LOG.warn("Exception during waiting for Log Search Config", e);
+ }
+ }
}
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
index 34e923f..0563d79 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
@@ -45,6 +45,7 @@ public class DocConstants {
public static final String FIELD_D = "Get values for particular field";
public static final String FORMAT_D = "File Export format, can be 'txt' or 'json'";
public static final String TOP = "Number that defines how many top element you would like to see.";
+ public static final String USER_D = "Filter for users (comma separated list)";
}
public class AuditOperationDescriptions {
@@ -101,6 +102,7 @@ public class DocConstants {
}
public class PublicOperationDescriptions {
+ public static final String GET_FEATURES_LIST = "Get features list.";
public static final String GET_APP_DETAILS_OD = "Get application details.";
public static final String GET_AUTH_DETAILS_OD = "Get authentication details.";
public static final String GET_ALL_PROPERTIES_INFO_OD = "List all available properties for Log Search and Log Feeder";
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
index 6c259cd..c3a5956 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/handler/CreateCollectionHandler.java
@@ -88,7 +88,7 @@ public class CreateCollectionHandler implements SolrZkRequestHandler<Boolean> {
// Default is true, because if the collection and shard is already there, then it will return true
boolean returnValue = true;
- List<String> shardsList = new ArrayList<String>();
+ List<String> shardsList = new ArrayList<>();
for (int i = 0; i < solrPropsConfig.getNumberOfShards(); i++) {
shardsList.add("shard" + i);
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java
index c17f24a..3aabdbf 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java
@@ -23,9 +23,11 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.conf.AuthPropsConfig;
import org.apache.ambari.logsearch.common.PropertyDescriptionStorage;
import org.apache.ambari.logsearch.common.ShipperConfigDescriptionStorage;
+import org.apache.ambari.logsearch.conf.LogSearchConfigApiConfig;
import org.apache.ambari.logsearch.model.response.PropertyDescriptionData;
import org.apache.ambari.logsearch.model.response.ShipperConfigDescriptionData;
import org.springframework.beans.factory.annotation.Value;
@@ -45,11 +47,13 @@ public class InfoManager extends JsonManagerBase {
@Value("${java.runtime.version}")
private String javaRuntimeVersion;
-
@Inject
private AuthPropsConfig authPropsConfig;
@Inject
+ private LogSearchConfigApiConfig logSearchConfigApiConfig;
+
+ @Inject
private PropertyDescriptionStorage propertyDescriptionStore;
@Inject
@@ -73,6 +77,13 @@ public class InfoManager extends JsonManagerBase {
return authMap;
}
+ public Map<String, Object> getFeaturesMap() {
+ Map<String, Object> featuresMap = new HashMap<>();
+ featuresMap.put(LogSearchConstants.AUTH_FEATURE_KEY, getAuthMap());
+ featuresMap.put(LogSearchConstants.SHIPPER_CONFIG_API_KEY, logSearchConfigApiConfig.isConfigApiEnabled());
+ return featuresMap;
+ }
+
public Map<String, List<PropertyDescriptionData>> getPropertyDescriptions() {
return propertyDescriptionStore.getPropertyDescriptions();
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
index 9ae8b9a..061cc2c 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
@@ -608,7 +608,7 @@ public class ServiceLogsManager extends ManagerBase<ServiceLogData, ServiceLogRe
public HostLogFilesResponse getHostLogFileData(HostLogFilesRequest request) {
SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class);
- QueryResponse queryResponse = serviceLogsSolrDao.process(facetQuery, "/service/logs/hostlogfiles");
+ QueryResponse queryResponse = serviceLogsSolrDao.process(facetQuery, "/service/logs/files");
return responseDataGenerator.generateHostLogFilesResponse(queryResponse);
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
index b19db72..5ca95e1 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java
@@ -92,21 +92,22 @@ public class ShipperConfigManager extends JsonManagerBase {
}
}
- public Map<String, Object> testShipperConfig(String shipperConfig, String logId, String testEntry, String clusterName) {
+ public Response testShipperConfig(Map<String, Object> shipperConfig, String logId, String testEntry, String clusterName) {
try {
- LSServerInputConfig inputConfigValidate = new ObjectMapper().readValue(shipperConfig, LSServerInputConfig.class);
+ String shipperConfigJsonStr = new ObjectMapper().writeValueAsString(shipperConfig);
+ LSServerInputConfig inputConfigValidate = new ObjectMapper().readValue(shipperConfigJsonStr, LSServerInputConfig.class);
Validator validator = Validation.buildDefaultValidatorFactory().getValidator();
Set<ConstraintViolation<LSServerInputConfig>> violations = validator.validate(inputConfigValidate);
if (!violations.isEmpty()) {
throw new IllegalArgumentException("Error validating shipper config:\n" + violations);
}
-
String globalConfigs = logSearchConfigConfigurer.getConfig().getGlobalConfigs(clusterName);
- LogEntryParseTester tester = new LogEntryParseTester(testEntry, shipperConfig, globalConfigs, logId);
- return tester.parse();
+ LogEntryParseTester tester = new LogEntryParseTester(testEntry, shipperConfigJsonStr, globalConfigs, logId);
+ Map<String, Object> resultEntrty = tester.parse();
+ return Response.ok().entity(resultEntrty).build();
} catch (Exception e) {
Map<String, Object> errorResponse = ImmutableMap.of("errorMessage", (Object)e.toString());
- return errorResponse;
+ return Response.serverError().entity(errorResponse).build();
}
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditLogRequest.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/ShipperConfigTestParams.java
similarity index 54%
copy from ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditLogRequest.java
copy to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/ShipperConfigTestParams.java
index 2abc69f..cb54607 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditLogRequest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/ShipperConfigTestParams.java
@@ -16,25 +16,30 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.logsearch.model.request.impl;
+package org.apache.ambari.logsearch.model.request;
+import io.swagger.annotations.ApiParam;
import org.apache.ambari.logsearch.common.LogSearchConstants;
-import org.apache.ambari.logsearch.model.request.FieldParamDefinition;
-import javax.ws.rs.QueryParam;
+import java.util.Map;
-public class FieldAuditLogRequest extends BaseLogRequest implements FieldParamDefinition {
+import static org.apache.ambari.logsearch.doc.DocConstants.CommonDescriptions.TOP;
- @QueryParam(LogSearchConstants.REQUEST_PARAM_FIELD)
- private String field;
+public interface ShipperConfigTestParams {
- @Override
- public String getField() {
- return field;
- }
+ Map<String, Object> getShipperConfig();
+
+ @ApiParam(value = TOP, name = LogSearchConstants.REQUEST_PARAM_SHIPPER_CONFIG, required = true)
+ void setShipperConfig(Map<String, Object> shipperConfig);
+
+ String getLogId();
+
+ @ApiParam(value = TOP, name = LogSearchConstants.REQUEST_PARAM_LOG_ID, required = true)
+ void setLogId(String logId);
+
+ String getTestEntry();
+
+ @ApiParam(value = TOP, name = LogSearchConstants.REQUEST_PARAM_TEST_ENTRY, required = true)
+ void setTestEntry(String testEntry);
- @Override
- public void setField(String field) {
- this.field = field;
- }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditServiceLoadRequest.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/UserParamDefinition.java
similarity index 66%
copy from ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditServiceLoadRequest.java
copy to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/UserParamDefinition.java
index 64ee2d2..38ffe5d 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditServiceLoadRequest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/UserParamDefinition.java
@@ -16,10 +16,18 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.ambari.logsearch.model.request.impl;
+package org.apache.ambari.logsearch.model.request;
-import org.apache.ambari.logsearch.common.Marker;
+import io.swagger.annotations.ApiParam;
+import org.apache.ambari.logsearch.common.LogSearchConstants;
+
+import static org.apache.ambari.logsearch.doc.DocConstants.CommonDescriptions.USER_D;
+
+public interface UserParamDefinition {
+
+ String getUserList();
+
+ @ApiParam(value = USER_D, name = LogSearchConstants.REQUEST_PARAM_USERS)
+ void setUserList(String userList);
-@Marker
-public class AuditServiceLoadRequest extends BaseLogRequest {
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditBarGraphRequest.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditBarGraphRequest.java
index 03ca32d..f135af5 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditBarGraphRequest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditBarGraphRequest.java
@@ -20,10 +20,14 @@ package org.apache.ambari.logsearch.model.request.impl;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.model.request.UnitParamDefinition;
+import org.apache.ambari.logsearch.model.request.UserParamDefinition;
import javax.ws.rs.QueryParam;
-public class AuditBarGraphRequest extends BaseLogRequest implements UnitParamDefinition {
+public class AuditBarGraphRequest extends BaseLogRequest implements UnitParamDefinition, UserParamDefinition {
+
+ @QueryParam(LogSearchConstants.REQUEST_PARAM_USERS)
+ private String userList;
@QueryParam(LogSearchConstants.REQUEST_PARAM_UNIT)
private String unit;
@@ -37,4 +41,14 @@ public class AuditBarGraphRequest extends BaseLogRequest implements UnitParamDef
public void setUnit(String unit) {
this.unit = unit;
}
+
+ @Override
+ public String getUserList() {
+ return userList;
+ }
+
+ @Override
+ public void setUserList(String userList) {
+ this.userList = userList;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditComponentRequest.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditComponentRequest.java
index 636c492..bae8a13 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditComponentRequest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditComponentRequest.java
@@ -18,8 +18,23 @@
*/
package org.apache.ambari.logsearch.model.request.impl;
+import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.Marker;
+import org.apache.ambari.logsearch.model.request.UserParamDefinition;
+
+import javax.ws.rs.QueryParam;
@Marker
-public class AuditComponentRequest extends BaseLogRequest {
+public class AuditComponentRequest extends BaseLogRequest implements UserParamDefinition {
+ @QueryParam(LogSearchConstants.REQUEST_PARAM_USERS)
+ private String userList;
+
+ @Override
+ public String getUserList() {
+ return userList;
+ }
+
+ public void setUserList(String userList) {
+ this.userList = userList;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditLogRequest.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditLogRequest.java
index 36fa378..799add4 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditLogRequest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditLogRequest.java
@@ -20,14 +20,18 @@ package org.apache.ambari.logsearch.model.request.impl;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.model.request.LastPageParamDefinition;
+import org.apache.ambari.logsearch.model.request.UserParamDefinition;
import javax.ws.rs.QueryParam;
-public class AuditLogRequest extends BaseLogRequest implements LastPageParamDefinition {
+public class AuditLogRequest extends BaseLogRequest implements LastPageParamDefinition, UserParamDefinition {
@QueryParam(LogSearchConstants.REQUEST_PARAM_LAST_PAGE)
private boolean isLastPage;
+ @QueryParam(LogSearchConstants.REQUEST_PARAM_USERS)
+ private String userList;
+
@Override
public boolean isLastPage() {
return isLastPage;
@@ -37,4 +41,14 @@ public class AuditLogRequest extends BaseLogRequest implements LastPageParamDefi
public void setLastPage(boolean lastPage) {
isLastPage = lastPage;
}
+
+ @Override
+ public String getUserList() {
+ return userList;
+ }
+
+ @Override
+ public void setUserList(String userList) {
+ this.userList = userList;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditServiceLoadRequest.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditServiceLoadRequest.java
index 64ee2d2..5f91320 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditServiceLoadRequest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditServiceLoadRequest.java
@@ -18,8 +18,25 @@
*/
package org.apache.ambari.logsearch.model.request.impl;
+import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.Marker;
+import org.apache.ambari.logsearch.model.request.UserParamDefinition;
+
+import javax.ws.rs.QueryParam;
@Marker
-public class AuditServiceLoadRequest extends BaseLogRequest {
+public class AuditServiceLoadRequest extends BaseLogRequest implements UserParamDefinition {
+
+ @QueryParam(LogSearchConstants.REQUEST_PARAM_USERS)
+ private String userList;
+
+ @Override
+ public String getUserList() {
+ return userList;
+ }
+
+ @Override
+ public void setUserList(String userList) {
+ this.userList = userList;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java
index b763830..738c24e 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java
@@ -25,6 +25,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import javax.annotation.Nullable;
+import javax.ws.rs.DefaultValue;
import javax.ws.rs.QueryParam;
public class CommonSearchRequest implements SearchRequest, CommonSearchParamDefinition {
@@ -33,9 +34,11 @@ public class CommonSearchRequest implements SearchRequest, CommonSearchParamDefi
private String startIndex;
@QueryParam(LogSearchConstants.REQUEST_PARAM_PAGE)
+ @DefaultValue(LogSearchConstants.REQUEST_PARAM_PAGE_DEFAULT_VALUE)
private String page;
@QueryParam(LogSearchConstants.REQUEST_PARAM_PAGE_SIZE)
+ @DefaultValue(LogSearchConstants.REQUEST_PARAM_PAGE_SIZE_DEFAULT_VALUE)
private String pageSize;
@QueryParam(LogSearchConstants.REQUEST_PARAM_SORT_BY)
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditBarGraphRequest.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditBarGraphRequest.java
index aa08aaf..6930698 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditBarGraphRequest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditBarGraphRequest.java
@@ -20,10 +20,14 @@ package org.apache.ambari.logsearch.model.request.impl;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.model.request.FieldParamDefinition;
+import org.apache.ambari.logsearch.model.request.UserParamDefinition;
import javax.ws.rs.QueryParam;
-public class FieldAuditBarGraphRequest extends AuditBarGraphRequest implements FieldParamDefinition {
+public class FieldAuditBarGraphRequest extends AuditBarGraphRequest implements FieldParamDefinition, UserParamDefinition {
+
+ @QueryParam(LogSearchConstants.REQUEST_PARAM_USERS)
+ private String userList;
@QueryParam(LogSearchConstants.REQUEST_PARAM_FIELD)
private String field;
@@ -37,4 +41,14 @@ public class FieldAuditBarGraphRequest extends AuditBarGraphRequest implements F
public void setField(String field) {
this.field = field;
}
+
+ @Override
+ public String getUserList() {
+ return userList;
+ }
+
+ @Override
+ public void setUserList(String userList) {
+ this.userList = userList;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditLogRequest.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditLogRequest.java
index 2abc69f..7f3e6bf 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditLogRequest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/FieldAuditLogRequest.java
@@ -20,14 +20,18 @@ package org.apache.ambari.logsearch.model.request.impl;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.model.request.FieldParamDefinition;
+import org.apache.ambari.logsearch.model.request.UserParamDefinition;
import javax.ws.rs.QueryParam;
-public class FieldAuditLogRequest extends BaseLogRequest implements FieldParamDefinition {
+public class FieldAuditLogRequest extends BaseLogRequest implements FieldParamDefinition, UserParamDefinition {
@QueryParam(LogSearchConstants.REQUEST_PARAM_FIELD)
private String field;
+ @QueryParam(LogSearchConstants.REQUEST_PARAM_USERS)
+ private String userList;
+
@Override
public String getField() {
return field;
@@ -37,4 +41,14 @@ public class FieldAuditLogRequest extends BaseLogRequest implements FieldParamDe
public void setField(String field) {
this.field = field;
}
+
+ @Override
+ public String getUserList() {
+ return userList;
+ }
+
+ @Override
+ public void setUserList(String userList) {
+ this.userList = userList;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditBarGraphRequest.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/ShipperConfigTestRequest.java
similarity index 51%
copy from ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditBarGraphRequest.java
copy to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/ShipperConfigTestRequest.java
index 03ca32d..0cd39ba 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/AuditBarGraphRequest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/ShipperConfigTestRequest.java
@@ -18,23 +18,48 @@
*/
package org.apache.ambari.logsearch.model.request.impl;
-import org.apache.ambari.logsearch.common.LogSearchConstants;
-import org.apache.ambari.logsearch.model.request.UnitParamDefinition;
+import org.apache.ambari.logsearch.model.request.ShipperConfigTestParams;
+import org.hibernate.validator.constraints.NotBlank;
-import javax.ws.rs.QueryParam;
+import java.util.Map;
-public class AuditBarGraphRequest extends BaseLogRequest implements UnitParamDefinition {
+public class ShipperConfigTestRequest implements ShipperConfigTestParams {
- @QueryParam(LogSearchConstants.REQUEST_PARAM_UNIT)
- private String unit;
+ @NotBlank
+ private String logId;
+
+ @NotBlank
+ private String testEntry;
+
+ @NotBlank
+ private Map<String, Object> shipperConfig;
@Override
- public String getUnit() {
- return unit;
+ public String getLogId() {
+ return logId;
}
@Override
- public void setUnit(String unit) {
- this.unit = unit;
+ public void setLogId(String logId) {
+ this.logId = logId;
+ }
+
+ @Override
+ public Map<String, Object> getShipperConfig() {
+ return shipperConfig;
+ }
+
+ @Override
+ public void setShipperConfig(Map<String, Object> shipperConfig) {
+ this.shipperConfig = shipperConfig;
+ }
+
+ @Override
+ public String getTestEntry() {
+ return testEntry;
+ }
+
+ public void setTestEntry(String testEntry) {
+ this.testEntry = testEntry;
}
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/TopFieldAuditLogRequest.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/TopFieldAuditLogRequest.java
index 4dd7c3e..6494822 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/TopFieldAuditLogRequest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/TopFieldAuditLogRequest.java
@@ -20,14 +20,19 @@ package org.apache.ambari.logsearch.model.request.impl;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.model.request.TopParamDefinition;
+import org.apache.ambari.logsearch.model.request.UserParamDefinition;
import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
-public class TopFieldAuditLogRequest extends FieldAuditLogRequest implements TopParamDefinition {
+public class TopFieldAuditLogRequest extends FieldAuditLogRequest implements TopParamDefinition, UserParamDefinition {
@PathParam(LogSearchConstants.REQUEST_PARAM_TOP)
private Integer top;
+ @QueryParam(LogSearchConstants.REQUEST_PARAM_USERS)
+ private String userList;
+
@Override
public Integer getTop() {
return top;
@@ -37,4 +42,14 @@ public class TopFieldAuditLogRequest extends FieldAuditLogRequest implements Top
public void setTop(Integer top) {
this.top = top;
}
+
+ @Override
+ public String getUserList() {
+ return userList;
+ }
+
+ @Override
+ public void setUserList(String userList) {
+ this.userList = userList;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/UserExportRequest.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/UserExportRequest.java
index 1583f94..3eabe79 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/UserExportRequest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/request/impl/UserExportRequest.java
@@ -20,14 +20,18 @@ package org.apache.ambari.logsearch.model.request.impl;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.model.request.FormatParamDefinition;
+import org.apache.ambari.logsearch.model.request.UserParamDefinition;
import javax.ws.rs.QueryParam;
-public class UserExportRequest extends FieldAuditLogRequest implements FormatParamDefinition {
+public class UserExportRequest extends FieldAuditLogRequest implements FormatParamDefinition, UserParamDefinition {
@QueryParam(LogSearchConstants.REQUEST_PARAM_FORMAT)
private String format;
+ @QueryParam(LogSearchConstants.REQUEST_PARAM_USERS)
+ private String userList;
+
@Override
public String getFormat() {
return format;
@@ -37,4 +41,14 @@ public class UserExportRequest extends FieldAuditLogRequest implements FormatPar
public void setFormat(String format) {
this.format = format;
}
+
+ @Override
+ public String getUserList() {
+ return userList;
+ }
+
+ @Override
+ public void setUserList(String userList) {
+ this.userList = userList;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java
index 9507c25..52ecdca 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java
@@ -38,6 +38,7 @@ import java.util.Map;
import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_ALL_PROPERTIES_INFO_OD;
import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_ALL_SHIPPER_CONFIG_INFO_OD;
import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_APP_DETAILS_OD;
+import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_FEATURES_LIST;
import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_LOGSEARCH_PROPERTIES_INFO_OD;
import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_AUTH_DETAILS_OD;
@@ -58,14 +59,6 @@ public class InfoResource {
}
@GET
- @Path("/auth")
- @Produces({"application/json"})
- @ApiOperation(GET_AUTH_DETAILS_OD)
- public Map<String, Boolean> getAuthInfo() {
- return infoManager.getAuthMap();
- }
-
- @GET
@Path("/properties")
@Produces({"application/json"})
@ApiOperation(GET_ALL_PROPERTIES_INFO_OD)
@@ -82,6 +75,22 @@ public class InfoResource {
}
@GET
+ @Path("/features")
+ @Produces({"application/json"})
+ @ApiOperation(GET_FEATURES_LIST)
+ public Map<String, Object> getFeatures() {
+ return infoManager.getFeaturesMap();
+ }
+
+ @GET
+ @Path("/features/auth")
+ @Produces({"application/json"})
+ @ApiOperation(GET_AUTH_DETAILS_OD)
+ public Map<String, Boolean> getAuthInfo() {
+ return infoManager.getAuthMap();
+ }
+
+ @GET
@Path("/shipperconfig")
@Produces({"application/json"})
@ApiOperation(GET_ALL_SHIPPER_CONFIG_INFO_OD)
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/ShipperConfigResource.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/ShipperConfigResource.java
index d8b1441..5cd2d3c 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/ShipperConfigResource.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/ShipperConfigResource.java
@@ -23,7 +23,6 @@ import javax.inject.Inject;
import javax.inject.Named;
import javax.validation.Valid;
import javax.validation.executable.ValidateOnExecution;
-import javax.ws.rs.FormParam;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
@@ -38,10 +37,10 @@ import io.swagger.annotations.ApiOperation;
import org.apache.ambari.logsearch.manager.ShipperConfigManager;
import org.apache.ambari.logsearch.model.common.LSServerInputConfig;
import org.apache.ambari.logsearch.model.common.LSServerLogLevelFilterMap;
+import org.apache.ambari.logsearch.model.request.impl.ShipperConfigTestRequest;
import org.springframework.context.annotation.Scope;
import java.util.List;
-import java.util.Map;
import static org.apache.ambari.logsearch.doc.DocConstants.ShipperConfigOperationDescriptions.GET_LOG_LEVEL_FILTER_OD;
import static org.apache.ambari.logsearch.doc.DocConstants.ShipperConfigOperationDescriptions.GET_SERVICE_NAMES_OD;
@@ -100,9 +99,8 @@ public class ShipperConfigResource {
@Path("/input/{clusterName}/test")
@Produces({"application/json"})
@ApiOperation(TEST_SHIPPER_CONFIG_OD)
- public Map<String, Object> testShipperConfig(@FormParam("shipper_config") String shipperConfig, @FormParam("log_id") String logId,
- @FormParam("test_entry") String testEntry, @PathParam("clusterName") String clusterName) {
- return shipperConfigManager.testShipperConfig(shipperConfig, logId, testEntry, clusterName);
+ public Response testShipperConfig(@Valid ShipperConfigTestRequest request, @PathParam("clusterName") String clusterName) {
+ return shipperConfigManager.testShipperConfig(request.getShipperConfig(), request.getLogId(), request.getTestEntry(), clusterName);
}
@GET
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogSearchConfigStateFilter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogSearchConfigStateFilter.java
index 9b6cdfe..aa1e20d 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogSearchConfigStateFilter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogSearchConfigStateFilter.java
@@ -47,13 +47,16 @@ public class LogSearchConfigStateFilter implements Filter {
private static final Logger LOG = LoggerFactory.getLogger(LogSearchConfigStateFilter.class);
private static final String CONFIG_NOT_AVAILABLE = "Configuration is not available";
+ private static final String CONFIG_API_DISABLED = "Configuration API is disabled";
private final RequestMatcher requestMatcher;
private final LogSearchConfigState logSearchConfigState;
+ private final boolean enabled;
- public LogSearchConfigStateFilter(RequestMatcher requestMatcher, LogSearchConfigState logSearchConfigState) {
+ public LogSearchConfigStateFilter(RequestMatcher requestMatcher, LogSearchConfigState logSearchConfigState, boolean enabled) {
this.requestMatcher = requestMatcher;
this.logSearchConfigState = logSearchConfigState;
+ this.enabled = enabled;
}
@Override
@@ -67,9 +70,10 @@ public class LogSearchConfigStateFilter implements Filter {
if (requestMatcher.matches(request)) {
VResponse errorResponse = getErrorResponse();
if (errorResponse != null) {
- LOG.info("{} request is filtered out: {}", request.getRequestURL(), errorResponse.getMsgDesc());
+ int statusCode = enabled ? 500 : 409;
+ LOG.info("{} request is filtered out: {}", request.getRequestURL(), errorResponse.getMsgDesc());
HttpServletResponse resp = (HttpServletResponse) servletResponse;
- resp.setStatus(500);
+ resp.setStatus(statusCode);
resp.setContentType("application/json");
resp.getWriter().print(createStringFromErrorMessageObject(errorResponse));
return;
@@ -80,6 +84,9 @@ public class LogSearchConfigStateFilter implements Filter {
}
private VResponse getErrorResponse() {
+ if (!enabled) {
+ return RESTErrorUtil.createMessageResponse(CONFIG_API_DISABLED, MessageEnums.CONFIGURATION_API_DISABLED);
+ }
if (!logSearchConfigState.isLogSearchConfigAvailable()) {
return RESTErrorUtil.createMessageResponse(CONFIG_NOT_AVAILABLE, MessageEnums.CONFIGURATION_NOT_AVAILABLE);
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchJWTFilter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchJWTFilter.java
index 164f646..fc5449b 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchJWTFilter.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchJWTFilter.java
@@ -67,6 +67,11 @@ public class LogsearchJWTFilter extends AbstractJWTFilter {
}
@Override
+ protected List<String> getUserAgentList() {
+ return authPropsConfig.getUserAgentList();
+ }
+
+ @Override
protected Collection<? extends GrantedAuthority> getAuthorities() {
return null; // TODO
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java
index b9e577c..1ac8b63 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java
@@ -38,6 +38,7 @@ public class AuditBarGraphRequestQueryConverterTest extends AbstractRequestConve
public void testConvert() {
// GIVEN
AuditBarGraphRequest request = new AuditBarGraphRequest();
+ request.setUserList("joe,steven");
// WHEN
fillBaseLogRequestWithTestData(request);
request.setUnit("+1HOUR");
@@ -45,7 +46,7 @@ public class AuditBarGraphRequestQueryConverterTest extends AbstractRequestConve
SolrQuery query = underTest.convert(request);
assertEquals("?q=*%3A*&facet=true&facet.pivot=%7B%21range%3Dr1%7Drepo&facet.mincount=1&facet.limit=-1&facet.sort=index" +
"&facet.range=%7B%21tag%3Dr1%7DevtTime&f.evtTime.facet.range.start=2016-09-13T22%3A00%3A01.000Z&f.evtTime.facet.range.end=2016-09-14T22%3A00%3A01.000Z&f.evtTime.facet.range.gap=%2B1HOUR" +
- "&rows=0&start=0&fq=cluster%3Acl1",
+ "&rows=0&start=0&fq=cluster%3Acl1&fq=reqUser%3A%28joe+OR+steven%29",
query.toQueryString());
}
diff --git a/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java b/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java
index 330a036..8e4ed07 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java
@@ -41,12 +41,13 @@ public class AuditLogRequestConverterTest extends AbstractRequestConverterTest {
// GIVEN
AuditLogRequest request = new AuditLogRequest();
fillBaseLogRequestWithTestData(request);
+ request.setUserList("joe,steven");
// WHEN
SimpleQuery simpleQuery = underTest.convert(request);
SolrQuery queryResult = new DefaultQueryParser().doConstructSolrQuery(simpleQuery);
// THEN
assertEquals("?q=*%3A*&start=0&rows=25&fq=repo%3A%28logsearch_app+secure_log%29&fq=-repo%3A%28hst_agent+system_message%29" +
- "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=cluster%3Acl1&sort=evtTime+desc%2Cseq_num+desc",
+ "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=cluster%3Acl1&fq=reqUser%3A%28joe+steven%29&sort=evtTime+desc%2Cseq_num+desc",
queryResult.toQueryString());
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/app.module.ts b/ambari-logsearch/ambari-logsearch-web/src/app/app.module.ts
index c6c6922..5a2b59d 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/app.module.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/app.module.ts
@@ -35,10 +35,10 @@ import {ServiceInjector} from '@app/classes/service-injector';
import {mockApiDataService} from '@app/services/mock-api-data.service'
import {HttpClientService} from '@app/services/http-client.service';
-import {ComponentActionsService} from '@app/services/component-actions.service';
import {UtilsService} from '@app/services/utils.service';
import {LogsContainerService} from '@app/services/logs-container.service';
import {ComponentGeneratorService} from '@app/services/component-generator.service';
+import {UserSettingsService} from '@app/services/user-settings.service';
import {AppSettingsService} from '@app/services/storage/app-settings.service';
import {AppStateService} from '@app/services/storage/app-state.service';
@@ -50,13 +50,13 @@ import {ServiceLogsTruncatedService} from '@app/services/storage/service-logs-tr
import {GraphsService} from '@app/services/storage/graphs.service';
import {HostsService} from '@app/services/storage/hosts.service';
import {UserConfigsService} from '@app/services/storage/user-configs.service';
-import {FiltersService} from '@app/services/storage/filters.service';
import {ClustersService} from '@app/services/storage/clusters.service';
import {ComponentsService} from '@app/services/storage/components.service';
import {ServiceLogsFieldsService} from '@app/services/storage/service-logs-fields.service';
import {AuditLogsFieldsService} from '@app/services/storage/audit-logs-fields.service';
import {TabsService} from '@app/services/storage/tabs.service';
import {AuthService} from '@app/services/auth.service';
+import {HistoryManagerService} from '@app/services/history-manager.service';
import {reducer} from '@app/services/storage/reducers.service';
import {AppComponent} from '@app/components/app.component';
@@ -96,6 +96,8 @@ import {GraphTooltipComponent} from '@app/components/graph-tooltip/graph-tooltip
import {GraphLegendItemComponent} from '@app/components/graph-legend-item/graph-legend-item.component';
import {TimeLineGraphComponent} from '@app/components/time-line-graph/time-line-graph.component';
import {ContextMenuComponent} from '@app/components/context-menu/context-menu.component';
+import {HistoryItemControlsComponent} from '@app/components/history-item-controls/history-item-controls.component';
+import {LogIndexFilterComponent} from '@app/components/log-index-filter/log-index-filter.component';
import {TimeZoneAbbrPipe} from '@app/pipes/timezone-abbr.pipe';
import {TimerSecondsPipe} from '@app/pipes/timer-seconds.pipe';
@@ -159,6 +161,8 @@ export function getXHRBackend(injector: Injector, browser: BrowserXhr, xsrf: XSR
GraphLegendItemComponent,
TimeLineGraphComponent,
ContextMenuComponent,
+ HistoryItemControlsComponent,
+ LogIndexFilterComponent,
TimeZoneAbbrPipe,
TimerSecondsPipe
],
@@ -183,10 +187,10 @@ export function getXHRBackend(injector: Injector, browser: BrowserXhr, xsrf: XSR
],
providers: [
HttpClientService,
- ComponentActionsService,
UtilsService,
LogsContainerService,
ComponentGeneratorService,
+ UserSettingsService,
AppSettingsService,
AppStateService,
AuditLogsService,
@@ -197,7 +201,6 @@ export function getXHRBackend(injector: Injector, browser: BrowserXhr, xsrf: XSR
GraphsService,
HostsService,
UserConfigsService,
- FiltersService,
ClustersService,
ComponentsService,
ServiceLogsFieldsService,
@@ -208,10 +211,14 @@ export function getXHRBackend(injector: Injector, browser: BrowserXhr, xsrf: XSR
useFactory: getXHRBackend,
deps: [Injector, BrowserXhr, XSRFStrategy, ResponseOptions]
},
- AuthService
+ AuthService,
+ HistoryManagerService
],
bootstrap: [AppComponent],
- entryComponents: [NodeBarComponent],
+ entryComponents: [
+ NodeBarComponent,
+ HistoryItemControlsComponent
+ ],
schemas: [CUSTOM_ELEMENTS_SCHEMA]
})
export class AppModule {
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/components/graph/graph.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/components/graph/graph.component.ts
index 0cfe69a..b9140cd 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/components/graph/graph.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/components/graph/graph.component.ts
@@ -122,20 +122,6 @@ export class GraphComponent implements AfterViewInit, OnChanges {
skipZeroValuesInTooltip: boolean = true;
/**
- * Indicates whether context menu for X axis ticks is available
- * @type {boolean}
- */
- @Input()
- hasXTickContextMenu: boolean = false;
-
- /**
- * Indicates whether context menu for Y axis ticks is available
- * @type {boolean}
- */
- @Input()
- hasYTickContextMenu: boolean = false;
-
- /**
* Indicates whether X axis event should be emitted with formatted string values that are displayed
* (instead of raw values)
* @type {boolean}
@@ -320,7 +306,7 @@ export class GraphComponent implements AfterViewInit, OnChanges {
this.xAxis = axis;
this.svg.append('g').attr('class', `axis ${this.xAxisClassName}`).attr('transform', `translate(0,${this.height})`)
.call(this.xAxis);
- if (this.hasXTickContextMenu) {
+ if (this.xTickContextMenu.observers.length) {
this.svg.selectAll(`.${this.xAxisClassName} .tick`).on('contextmenu', (tickValue: any, index: number): void => {
const tick = this.emitFormattedXTick ? this.xAxisTickFormatter(tickValue, index) : tickValue,
nativeEvent = d3.event;
@@ -342,7 +328,7 @@ export class GraphComponent implements AfterViewInit, OnChanges {
}
this.yAxis = axis;
this.svg.append('g').attr('class', `axis ${this.yAxisClassName}`).call(this.yAxis);
- if (this.hasYTickContextMenu) {
+ if (this.yTickContextMenu.observers.length) {
this.svg.selectAll(`.${this.yAxisClassName} .tick`).on('contextmenu', (tickValue: any, index: number): void => {
const tick = this.emitFormattedYTick ? this.yAxisTickFormatter(tickValue, index): tickValue,
nativeEvent = d3.event;
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/filtering.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/filtering.ts
index 3348969..bb75786 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/filtering.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/filtering.ts
@@ -48,7 +48,7 @@ export interface SortingListItem extends ListItem {
export interface FilterCondition {
label?: string;
options?: (ListItem | TimeUnitListItem[])[];
- defaultSelection?: ListItem | ListItem[] | number;
+ defaultSelection?: ListItem | ListItem[] | number | boolean;
iconClass?: string;
fieldName?: string;
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/list-item.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/list-item.ts
index 1aaaecc..8505373 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/list-item.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/list-item.ts
@@ -17,10 +17,11 @@
*/
export interface ListItem {
- id?: string;
+ id?: string | number;
label?: string;
value: any;
iconClass?: string;
isChecked?: boolean;
- action?: string;
+ onSelect?: Function;
+ isDivider?: boolean;
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/app-settings.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/app-settings.ts
index 11821a3..3ba5089 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/app-settings.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/app-settings.ts
@@ -17,11 +17,15 @@
*/
import * as moment from 'moment-timezone';
+import {HomogeneousObject} from '@app/classes/object';
+import {Filter} from '@app/classes/models/filter';
export interface AppSettings {
timeZone: string;
+ logIndexFilters: HomogeneousObject<HomogeneousObject<Filter>>;
}
export const defaultSettings: AppSettings = {
- timeZone: moment.tz.guess()
-}
+ timeZone: moment.tz.guess(),
+ logIndexFilters: {}
+};
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/app-state.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/app-state.ts
index c3279ce..b187ca6 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/app-state.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/app-state.ts
@@ -17,8 +17,14 @@
*/
import {ActiveServiceLogEntry} from '@app/classes/active-service-log-entry';
+import {ListItem} from '@app/classes/list-item';
import {LogsType} from '@app/classes/string';
+export interface History {
+ items: ListItem[];
+ currentId: number;
+}
+
export interface AppState {
isAuthorized: boolean;
isInitialLoading: boolean;
@@ -28,6 +34,7 @@ export interface AppState {
isServiceLogContextView: boolean;
activeLog: ActiveServiceLogEntry | null;
activeFilters: object;
+ history: History;
}
export const initialState: AppState = {
@@ -38,5 +45,9 @@ export const initialState: AppState = {
isServiceLogsFileView: false,
isServiceLogContextView: false,
activeLog: null,
- activeFilters: null
+ activeFilters: null,
+ history: {
+ items: [],
+ currentId: -1
+ }
};
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/filter.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/filter.ts
index c7ff662..b3e01aa 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/filter.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/filter.ts
@@ -16,10 +16,12 @@
* limitations under the License.
*/
+import {LogLevel} from '@app/classes/string';
+
export interface Filter {
label: string;
hosts: string[];
- defaultLevels: string[];
- overrideLevels: string[];
- expiryTime: string;
+ defaultLevels: LogLevel[];
+ overrideLevels: LogLevel[];
+ expiryTime: string | null;
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/store.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/store.ts
index f2d0696..eb87714 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/store.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/store.ts
@@ -16,6 +16,7 @@
* limitations under the License.
*/
+import {ReflectiveInjector} from '@angular/core';
import {Observable} from 'rxjs/Observable';
import {Store, Action} from '@ngrx/store';
import {AppSettings} from '@app/classes/models/app-settings';
@@ -26,21 +27,25 @@ import {BarGraph} from '@app/classes/models/bar-graph';
import {Graph} from '@app/classes/models/graph';
import {NodeItem} from '@app/classes/models/node-item';
import {UserConfig} from '@app/classes/models/user-config';
-import {Filter} from '@app/classes/models/filter';
import {AuditLogField} from '@app/classes/models/audit-log-field';
import {ServiceLogField} from '@app/classes/models/service-log-field';
import {Tab} from '@app/classes/models/tab';
-
-export const storeActions = {
- 'ARRAY.ADD': 'ADD',
- 'ARRAY.ADD.START': 'ADD_TO_START',
- 'ARRAY.DELETE.PRIMITIVE': 'DELETE_PRIMITIVE',
- 'ARRAY.DELETE.OBJECT': 'DELETE_OBJECT',
- 'ARRAY.CLEAR': 'CLEAR',
- 'ARRAY.MAP': 'MAP',
-
- 'OBJECT.SET': 'SET'
-};
+import {UtilsService} from '@app/services/utils.service';
+
+const storeActions = {
+ 'ARRAY.ADD': 'ADD',
+ 'ARRAY.ADD.START': 'ADD_TO_START',
+ 'ARRAY.ADD.UNIQUE': 'ADD_UNIQUE',
+ 'ARRAY.DELETE.PRIMITIVE': 'DELETE_PRIMITIVE',
+ 'ARRAY.DELETE.OBJECT': 'DELETE_OBJECT',
+ 'ARRAY.CLEAR': 'CLEAR',
+ 'ARRAY.MAP': 'MAP',
+
+ 'OBJECT.SET': 'SET'
+ },
+ provider = ReflectiveInjector.resolve([UtilsService]),
+ injector = ReflectiveInjector.fromResolvedProviders(provider),
+ utils = injector.get(UtilsService);
export interface AppStore {
appSettings: AppSettings;
@@ -53,7 +58,6 @@ export interface AppStore {
graphs: Graph[];
hosts: NodeItem[];
userConfigs: UserConfig[];
- filters: Filter[];
clusters: string[];
components: NodeItem[];
serviceLogsFields: ServiceLogField[];
@@ -98,6 +102,13 @@ export class CollectionModelService extends ModelService {
});
}
+ addUniqueInstances(instances: any[]): void {
+ this.store.dispatch({
+ type: `${storeActions['ARRAY.ADD.UNIQUE']}_${this.modelName}`,
+ payload: instances
+ });
+ }
+
deleteObjectInstance(instance: any): void {
this.store.dispatch({
type: `${storeActions['ARRAY.DELETE.OBJECT']}_${this.modelName}`,
@@ -144,9 +155,9 @@ export class ObjectModelService extends ModelService {
}
setParameter(key: string, value: any): void {
- let payload = {};
- payload[key] = value;
- this.setParameters(payload);
+ this.setParameters({
+ [key]: value
+ });
}
setParameters(params: any): void {
@@ -165,6 +176,8 @@ export function getCollectionReducer(modelName: string, defaultState: any = []):
return [...state, ...action.payload];
case `${storeActions['ARRAY.ADD.START']}_${modelName}`:
return [...action.payload, ...state];
+ case `${storeActions['ARRAY.ADD.UNIQUE']}_${modelName}`:
+ return utils.pushUniqueValues(state.slice(), action.payload);
case `${storeActions['ARRAY.DELETE.OBJECT']}_${modelName}`:
return state.filter(instance => instance.id !== action.payload.id);
case `${storeActions['ARRAY.DELETE.PRIMITIVE']}_${modelName}`:
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/object.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/object.ts
index 4d0c7f6..13cc4cc 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/object.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/object.ts
@@ -16,4 +16,12 @@
* limitations under the License.
*/
+import {LogLevel} from '@app/classes/string';
+
export type HomogeneousObject<T> = {[key: string]: T};
+
+export interface LogLevelObject {
+ name: LogLevel;
+ label: string;
+ color: string;
+}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-query-params.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-query-params.ts
index dc82b9e..e9c0149 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-query-params.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-query-params.ts
@@ -16,35 +16,8 @@
* limitations under the License.
*/
-import {QueryParams} from '@app/classes/queries/query-params';
-import {SortingType} from '@app/classes/string';
+import {LogsListQueryParams} from '@app/classes/queries/logs-query-params';
-export const defaultParams = {
- page: '0',
- pageSize: '10'
-};
-
-export class AuditLogsQueryParams extends QueryParams {
- clusters?: string;
- mustBe?: string;
- mustNot?: string;
- includeQuery?: string;
- excludeQuery?: string;
- from?: string;
- to?: string;
-}
-
-export class AuditLogsListQueryParams extends AuditLogsQueryParams {
- constructor(options: AuditLogsListQueryParams) {
- let finalParams = Object.assign({}, defaultParams, options);
- const page = parseInt(finalParams.page),
- pageSize = parseInt(finalParams.pageSize);
- finalParams.startIndex = isNaN(page) || isNaN(pageSize) ? '' : (page * pageSize).toString();
- super(finalParams);
- }
- page: string;
- pageSize: string;
- startIndex: string;
- sortBy?: string;
- sortType?: SortingType;
+export class AuditLogsListQueryParams extends LogsListQueryParams {
+ userList?: string;
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-top-resources-query-params.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-top-resources-query-params.ts
index 0d12539..d5e9cc5 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-top-resources-query-params.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-top-resources-query-params.ts
@@ -16,8 +16,8 @@
* limitations under the License.
*/
-import {AuditLogsQueryParams} from '@app/classes/queries/audit-logs-query-params';
+import {LogsQueryParams} from '@app/classes/queries/logs-query-params';
-export class AuditLogsTopResourcesQueryParams extends AuditLogsQueryParams {
+export class AuditLogsTopResourcesQueryParams extends LogsQueryParams {
field: string;
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-query-params.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/logs-query-params.ts
similarity index 83%
copy from ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-query-params.ts
copy to ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/logs-query-params.ts
index dc82b9e..9e7f5d0 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-query-params.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/logs-query-params.ts
@@ -16,15 +16,15 @@
* limitations under the License.
*/
-import {QueryParams} from '@app/classes/queries/query-params';
import {SortingType} from '@app/classes/string';
+import {QueryParams} from '@app/classes/queries/query-params';
-export const defaultParams = {
+const defaultListParams = {
page: '0',
pageSize: '10'
};
-export class AuditLogsQueryParams extends QueryParams {
+export class LogsQueryParams extends QueryParams {
clusters?: string;
mustBe?: string;
mustNot?: string;
@@ -34,9 +34,9 @@ export class AuditLogsQueryParams extends QueryParams {
to?: string;
}
-export class AuditLogsListQueryParams extends AuditLogsQueryParams {
- constructor(options: AuditLogsListQueryParams) {
- let finalParams = Object.assign({}, defaultParams, options);
+export class LogsListQueryParams extends LogsQueryParams {
+ constructor(options: LogsListQueryParams) {
+ let finalParams = Object.assign({}, defaultListParams, options);
const page = parseInt(finalParams.page),
pageSize = parseInt(finalParams.pageSize);
finalParams.startIndex = isNaN(page) || isNaN(pageSize) ? '' : (page * pageSize).toString();
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/service-logs-query-params.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/service-logs-query-params.ts
index 60c3d5c..ad88449 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/service-logs-query-params.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/service-logs-query-params.ts
@@ -16,9 +16,9 @@
* limitations under the License.
*/
-import {AuditLogsListQueryParams} from '@app/classes/queries/audit-logs-query-params';
+import {LogsListQueryParams} from '@app/classes/queries/logs-query-params';
-export class ServiceLogsQueryParams extends AuditLogsListQueryParams {
+export class ServiceLogsQueryParams extends LogsListQueryParams {
level?: string;
file_name?: string;
bundle_id?: string;
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/service-logs-truncated-query-params.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/service-logs-truncated-query-params.ts
index 3b08e11..071a443 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/service-logs-truncated-query-params.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/service-logs-truncated-query-params.ts
@@ -19,7 +19,7 @@
import {QueryParams} from '@app/classes/queries/query-params';
import {ScrollType} from '@app/classes/string';
-export const defaultParams = {
+const defaultParams = {
numberRows: '10',
scrollType: ''
};
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/filter.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/settings.ts
similarity index 72%
copy from ambari-logsearch/ambari-logsearch-web/src/app/classes/models/filter.ts
copy to ambari-logsearch/ambari-logsearch-web/src/app/classes/settings.ts
index c7ff662..dcef457 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/models/filter.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/settings.ts
@@ -16,10 +16,17 @@
* limitations under the License.
*/
-export interface Filter {
+import {HomogeneousObject} from '@app/classes/object';
+
+export type LevelOverridesConfig = HomogeneousObject<{
+ defaults: boolean;
+ overrides: boolean;
+}>
+
+export type LogIndexFilterComponentConfig = LevelOverridesConfig & {
+ name: string;
label: string;
- hosts: string[];
- defaultLevels: string[];
- overrideLevels: string[];
- expiryTime: string;
+ hosts: string;
+ expiryTime: string | null;
+ hasOverrides?: boolean;
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/string.ts b/ambari-logsearch/ambari-logsearch-web/src/app/classes/string.ts
index 21ff4ca..a8156e6 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/string.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/classes/string.ts
@@ -23,3 +23,5 @@ export type TimeRangeType = 'CURRENT' | 'LAST' | 'PAST';
export type SortingType = 'asc' | 'desc';
export type ScrollType = 'before' | 'after' | '';
+
+export type LogLevel = 'FATAL' | 'ERROR' | 'WARN' | 'INFO' | 'DEBUG' | 'TRACE' | 'UNKNOWN';
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.html
index ab6326a..b2bf90f 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.html
@@ -14,7 +14,30 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<menu-button *ngFor="let item of items" label="{{item.label | translate}}" [action]="item.action"
- [iconClass]="item.iconClass" [labelClass]="item.labelClass" [subItems]="item.subItems"
- [hideCaret]="item.hideCaret" [badge]="item.badge" [isRightAlign]="item.isRightAlign">
-</menu-button>
+
+<!-- TODO use listClass="history-dropdown" for custom styling -->
+<menu-button label="{{'topMenu.undo' | translate}}" [subItems]="undoItems" iconClass="fa fa-arrow-left"
+ listClass="history-dropdown" (buttonClick)="undoLatest()" (selectItem)="undo($event)"></menu-button>
+<menu-button label="{{'topMenu.redo' | translate}}" [subItems]="redoItems" iconClass="fa fa-arrow-right"
+ listClass="history-dropdown" (buttonClick)="redoLatest()" (selectItem)="redo($event)"></menu-button>
+<menu-button label="{{'topMenu.history' | translate}}" [subItems]="historyItems" iconClass="fa fa-history"
+ listClass="history-dropdown" [isRightAlign]="true"
+ additionalLabelComponentSetter="getHistoryItemIcons"></menu-button>
+<menu-button label="{{'topMenu.filter' | translate}}" iconClass="fa fa-filter"
+ (buttonClick)="openLogIndexFilter()"></menu-button>
+<menu-button *ngIf="!captureSeconds" label="{{'filter.capture' | translate}}" iconClass="fa fa-caret-right"
+ (buttonClick)="startCapture()"></menu-button>
+<menu-button *ngIf="captureSeconds" label="{{captureSeconds | timerSeconds}}" iconClass="fa fa-stop stop-icon"
+ (buttonClick)="stopCapture()"></menu-button>
+<menu-button label="{{'topMenu.refresh' | translate}}" iconClass="fa fa-refresh"
+ (buttonClick)="refresh()"></menu-button>
+<modal *ngIf="isLogIndexFilterDisplayed" (submit)="saveLogIndexFilter()" (cancel)="closeLogIndexFilter()"
+ (close)="closeLogIndexFilter()" [isExtraLargeModal]="true" [isSubmitDisabled]="isModalSubmitDisabled"
+ title="{{'logIndexFilter.title' | translate}}" submitButtonLabel="{{'modal.save' | translate}}">
+ <ng-template>
+ <form [formGroup]="settingsForm">
+ <log-index-filter formControlName="logIndexFilter"
+ (changeIsSubmitDisabled)="setModalSubmitDisabled($event)"></log-index-filter>
+ </form>
+ </ng-template>
+</modal>
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.less b/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.less
index fff57df..b87f447 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.less
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.less
@@ -15,10 +15,15 @@
* limitations under the License.
*/
+@import '../variables';
+
:host {
display: block;
margin-left: auto;
menu-button {
margin: 0 1em;
+ /deep/ .stop-icon {
+ color: @exclude-color;
+ }
}
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.spec.ts
index 081304e..3ec5429 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.spec.ts
@@ -18,7 +18,31 @@
import {CUSTOM_ELEMENTS_SCHEMA} from '@angular/core';
import {async, ComponentFixture, TestBed} from '@angular/core/testing';
+import {FormsModule, ReactiveFormsModule} from '@angular/forms';
import {TranslationModules} from '@app/test-config.spec';
+import {StoreModule} from '@ngrx/store';
+import {AuditLogsService, auditLogs} from '@app/services/storage/audit-logs.service';
+import {ServiceLogsService, serviceLogs} from '@app/services/storage/service-logs.service';
+import {AuditLogsFieldsService, auditLogsFields} from '@app/services/storage/audit-logs-fields.service';
+import {AuditLogsGraphDataService, auditLogsGraphData} from '@app/services/storage/audit-logs-graph-data.service';
+import {ServiceLogsFieldsService, serviceLogsFields} from '@app/services/storage/service-logs-fields.service';
+import {
+ ServiceLogsHistogramDataService, serviceLogsHistogramData
+} from '@app/services/storage/service-logs-histogram-data.service';
+import {AppSettingsService, appSettings} from '@app/services/storage/app-settings.service';
+import {AppStateService, appState} from '@app/services/storage/app-state.service';
+import {ClustersService, clusters} from '@app/services/storage/clusters.service';
+import {ComponentsService, components} from '@app/services/storage/components.service';
+import {HostsService, hosts} from '@app/services/storage/hosts.service';
+import {ServiceLogsTruncatedService, serviceLogsTruncated} from '@app/services/storage/service-logs-truncated.service';
+import {TabsService, tabs} from '@app/services/storage/tabs.service';
+import {HistoryManagerService} from '@app/services/history-manager.service';
+import {HttpClientService} from '@app/services/http-client.service';
+import {LogsContainerService} from '@app/services/logs-container.service';
+import {UserSettingsService} from '@app/services/user-settings.service';
+import {UtilsService} from '@app/services/utils.service';
+import {ModalComponent} from '@app/components/modal/modal.component';
+import {TimerSecondsPipe} from '@app/pipes/timer-seconds.pipe';
import {ActionMenuComponent} from './action-menu.component';
@@ -27,9 +51,63 @@ describe('ActionMenuComponent', () => {
let fixture: ComponentFixture<ActionMenuComponent>;
beforeEach(async(() => {
+ const httpClient = {
+ get: () => {
+ return {
+ subscribe: () => {
+ }
+ }
+ }
+ };
TestBed.configureTestingModule({
- imports: TranslationModules,
- declarations: [ActionMenuComponent],
+ imports: [
+ FormsModule,
+ ReactiveFormsModule,
+ ...TranslationModules,
+ StoreModule.provideStore({
+ auditLogs,
+ serviceLogs,
+ auditLogsFields,
+ auditLogsGraphData,
+ serviceLogsFields,
+ serviceLogsHistogramData,
+ appSettings,
+ appState,
+ clusters,
+ components,
+ hosts,
+ serviceLogsTruncated,
+ tabs
+ })
+ ],
+ declarations: [
+ ActionMenuComponent,
+ ModalComponent,
+ TimerSecondsPipe
+ ],
+ providers: [
+ {
+ provide: HttpClientService,
+ useValue: httpClient
+ },
+ HistoryManagerService,
+ LogsContainerService,
+ UserSettingsService,
+ UtilsService,
+ AuditLogsService,
+ ServiceLogsService,
+ AuditLogsFieldsService,
+ AuditLogsGraphDataService,
+ ServiceLogsFieldsService,
+ ServiceLogsHistogramDataService,
+ AppSettingsService,
+ AppStateService,
+ ClustersService,
+ ComponentsService,
+ HostsService,
+ ServiceLogsTruncatedService,
+ TabsService
+ ],
schemas: [CUSTOM_ELEMENTS_SCHEMA]
})
.compileComponents();
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.ts
index 72037f8..6b02843 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.ts
@@ -17,6 +17,11 @@
*/
import {Component} from '@angular/core';
+import {FormGroup} from '@angular/forms';
+import {LogsContainerService} from '@app/services/logs-container.service';
+import {HistoryManagerService} from '@app/services/history-manager.service';
+import {UserSettingsService} from '@app/services/user-settings.service';
+import {ListItem} from '@app/classes/list-item';
@Component({
selector: 'action-menu',
@@ -25,77 +30,77 @@ import {Component} from '@angular/core';
})
export class ActionMenuComponent {
- //TODO implement loading of real data into subItems
- readonly items = [
- {
- iconClass: 'fa fa-arrow-left',
- label: 'topMenu.undo',
- action: 'undo',
- subItems: [
- {
- label: 'Apply \'Last week\' filter'
- },
- {
- label: 'Clear all filters'
- },
- {
- label: 'Apply \'HDFS\' filter'
- },
- {
- label: 'Apply \'Errors\' filter'
- }
- ]
- },
- {
- iconClass: 'fa fa-arrow-right',
- label: 'topMenu.redo',
- action: 'redo',
- subItems: [
- {
- label: 'Apply \'Warnings\' filter'
- },
- {
- label: 'Switch to graph mode'
- },
- {
- label: 'Apply \'Custom Date\' filter'
- }
- ]
- },
- {
- iconClass: 'fa fa-refresh',
- label: 'topMenu.refresh',
- action: 'refresh'
- },
- {
- iconClass: 'fa fa-history',
- label: 'topMenu.history',
- action: 'openHistory',
- isRightAlign: true,
- subItems: [
- {
- label: 'Apply \'Custom Date\' filter'
- },
- {
- label: 'Switch to graph mode'
- },
- {
- label: 'Apply \'Warnings\' filter'
- },
- {
- label: 'Apply \'Last week\' filter'
- },
- {
- label: 'Clear all filters'
- },
- {
- label: 'Apply \'HDFS\' filter'
- },
- {
- label: 'Apply \'Errors\' filter'
- }
- ]
- }
- ];
+ constructor(
+ private logsContainer: LogsContainerService, private historyManager: HistoryManagerService,
+ private settings: UserSettingsService
+ ) {
+ }
+
+ get undoItems(): ListItem[] {
+ return this.historyManager.undoItems;
+ }
+
+ get redoItems(): ListItem[] {
+ return this.historyManager.redoItems;
+ }
+
+ get historyItems(): ListItem[] {
+ return this.historyManager.activeHistory;
+ }
+
+ get captureSeconds(): number {
+ return this.logsContainer.captureSeconds;
+ }
+
+ isLogIndexFilterDisplayed: boolean = false;
+
+ settingsForm: FormGroup = this.settings.settingsFormGroup;
+
+ isModalSubmitDisabled: boolean = true;
+
+ setModalSubmitDisabled(isDisabled: boolean): void {
+ this.isModalSubmitDisabled = isDisabled;
+ }
+
+ undoLatest(): void {
+ this.historyManager.undo(this.undoItems[0]);
+ }
+
+ redoLatest(): void {
+ this.historyManager.redo(this.redoItems[0]);
+ }
+
+ undo(item: ListItem): void {
+ this.historyManager.undo(item);
+ }
+
+ redo(item: ListItem): void {
+ this.historyManager.redo(item);
+ }
+
+ refresh(): void {
+ this.logsContainer.loadLogs();
+ }
+
+ openLogIndexFilter(): void {
+ this.isLogIndexFilterDisplayed = true;
+ }
+
+ closeLogIndexFilter(): void {
+ this.isLogIndexFilterDisplayed = false;
+ }
+
+ saveLogIndexFilter(): void {
+ this.isLogIndexFilterDisplayed = false;
+ this.settings.saveIndexFilterConfig();
+ }
+
+ startCapture(): void {
+ this.logsContainer.startCaptureTimer();
+ }
+
+ stopCapture(): void {
+ this.logsContainer.stopCaptureTimer();
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-entries/audit-logs-entries.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-entries/audit-logs-entries.component.html
index 22deef1..6ebb92e 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-entries/audit-logs-entries.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-entries/audit-logs-entries.component.html
@@ -25,8 +25,8 @@
svgId="top-users-graph"></horizontal-histogram>
</collapsible-panel>
<collapsible-panel commonTitle="{{'logs.topResources' | translate: resourcesGraphTitleParams}}" class="col-md-6">
- <horizontal-histogram [data]="topResourcesGraphData" [allowFractionalXTicks]="false" [hasYTickContextMenu]="true"
- [emitFormattedYTick]="true" (yTickContextMenu)="showContextMenu($event)"
+ <horizontal-histogram [data]="topResourcesGraphData" [allowFractionalXTicks]="false" [emitFormattedYTick]="true"
+ (yTickContextMenu)="showContextMenu($event)"
svgId="top-resources-graph"></horizontal-histogram>
<context-menu [isDisplayed]="isContextMenuDisplayed" [contextMenuItems]="contextMenuItems"
[leftPosition]="contextMenuLeft" [topPosition]="contextMenuTop" (itemSelect)="updateQuery($event)"
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-entries/audit-logs-entries.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-entries/audit-logs-entries.component.spec.ts
index f7d0cdc..51aaaa1 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-entries/audit-logs-entries.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-entries/audit-logs-entries.component.spec.ts
@@ -37,6 +37,7 @@ import {TabsService, tabs} from '@app/services/storage/tabs.service';
import {TabsComponent} from '@app/components/tabs/tabs.component';
import {LogsContainerService} from '@app/services/logs-container.service';
import {HttpClientService} from '@app/services/http-client.service';
+import {UtilsService} from '@app/services/utils.service';
import {AuditLogsEntriesComponent} from './audit-logs-entries.component';
@@ -82,6 +83,7 @@ describe('AuditLogsEntriesComponent', () => {
provide: HttpClientService,
useValue: httpClient
},
+ UtilsService,
AuditLogsService,
ServiceLogsService,
AuditLogsFieldsService,
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-table/audit-logs-table.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-table/audit-logs-table.component.html
index cad09bc..f970726 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-table/audit-logs-table.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-table/audit-logs-table.component.html
@@ -15,13 +15,12 @@
limitations under the License.
-->
-<dropdown-button class="pull-right" label="{{'logs.columns' | translate}}" [options]="columns" [isRightAlign]="true"
- [isMultipleChoice]="true" action="updateSelectedColumns"
- [additionalArgs]="logsTypeMapObject.fieldsModel"></dropdown-button>
+<dropdown-button class="pull-right" label="{{'logs.columns' | translate}}" (selectItem)="updateSelectedColumns($event)"
+ [options]="columns" [isRightAlign]="true" [isMultipleChoice]="true"></dropdown-button>
<form *ngIf="logs && logs.length" [formGroup]="filtersForm" class="row pull-right">
<filter-dropdown class="col-md-12" label="{{filters.auditLogsSorting.label | translate}}"
formControlName="auditLogsSorting" [options]="filters.auditLogsSorting.options"
- [isRightAlign]="true"></filter-dropdown>
+ [isRightAlign]="true" [showCommonLabelWithSelection]="true"></filter-dropdown>
</form>
<div class="panel panel-default">
<div class="panel-body">
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-table/audit-logs-table.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-table/audit-logs-table.component.ts
index deca936..fa5b1c5 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-table/audit-logs-table.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/audit-logs-table/audit-logs-table.component.ts
@@ -36,9 +36,7 @@ export class AuditLogsTableComponent extends LogsTableComponent {
readonly timeFormat: string = 'YYYY-MM-DD HH:mm:ss,SSS';
- get logsTypeMapObject(): object {
- return this.logsContainer.logsTypeMap.auditLogs;
- }
+ private readonly logsType: string = 'auditLogs';
get filters(): any {
return this.logsContainer.filters;
@@ -52,4 +50,8 @@ export class AuditLogsTableComponent extends LogsTableComponent {
return this.columns.find((column: ListItem): boolean => column.value === name);
}
+ updateSelectedColumns(columns: string[]): void {
+ this.logsContainer.updateSelectedColumns(columns, this.logsType);
+ }
+
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/context-menu/context-menu.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/context-menu/context-menu.component.spec.ts
index 1c5154b..1881f57 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/context-menu/context-menu.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/context-menu/context-menu.component.spec.ts
@@ -39,8 +39,8 @@ import {TabsService, tabs} from '@app/services/storage/tabs.service';
import {ComponentGeneratorService} from '@app/services/component-generator.service';
import {LogsContainerService} from '@app/services/logs-container.service';
import {HttpClientService} from '@app/services/http-client.service';
-import {ComponentActionsService} from '@app/services/component-actions.service';
import {AuthService} from '@app/services/auth.service';
+import {UtilsService} from '@app/services/utils.service';
import {DropdownListComponent} from '@app/components/dropdown-list/dropdown-list.component';
import {ContextMenuComponent} from './context-menu.component';
@@ -90,7 +90,6 @@ describe('ContextMenuComponent', () => {
provide: HttpClientService,
useValue: httpClient
},
- ComponentActionsService,
HostsService,
AuditLogsService,
ServiceLogsService,
@@ -104,7 +103,8 @@ describe('ContextMenuComponent', () => {
ComponentsService,
ServiceLogsTruncatedService,
TabsService,
- AuthService
+ AuthService,
+ UtilsService
],
schemas: [CUSTOM_ELEMENTS_SCHEMA]
})
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/date-picker/date-picker.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/date-picker/date-picker.component.ts
index e33d71e..93ebe37 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/date-picker/date-picker.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/date-picker/date-picker.component.ts
@@ -20,7 +20,7 @@ import {
Component, OnInit, OnChanges, OnDestroy, SimpleChanges, Input, Output, EventEmitter, ViewChild, ElementRef
} from '@angular/core';
import * as $ from 'jquery';
-import {Moment} from 'moment-timezone';
+import * as moment from 'moment';
import '@vendor/js/bootstrap-datetimepicker.min';
import {AppSettingsService} from '@app/services/storage/app-settings.service';
@@ -31,16 +31,16 @@ import {AppSettingsService} from '@app/services/storage/app-settings.service';
export class DatePickerComponent implements OnInit, OnChanges, OnDestroy {
constructor(private appSettings: AppSettingsService) {
- appSettings.getParameter('timeZone').subscribe((value: string): void => {
+ }
+
+ ngOnInit(): void {
+ this.appSettings.getParameter('timeZone').subscribe((value: string): void => {
this.destroyDatePicker();
this.timeZone = value;
if (this.datePickerElement) {
this.createDatePicker();
}
});
- }
-
- ngOnInit(): void {
this.createDatePicker();
}
@@ -56,10 +56,10 @@ export class DatePickerComponent implements OnInit, OnChanges, OnDestroy {
/**
* Value of time input field passed from parent component
- * @type {Moment}
+ * @type {Moment|Date|string}
*/
@Input()
- time: Moment;
+ time: moment.Moment | Date | string;
@Output()
timeChange: EventEmitter<number> = new EventEmitter();
@@ -89,10 +89,11 @@ export class DatePickerComponent implements OnInit, OnChanges, OnDestroy {
/**
* Set value to time input field
- * @param {Moment} time
+ * @param {Moment|Date|string} time
*/
- private setTime(time: Moment): void {
- this.datePickerElement.data('DateTimePicker').date(time);
+ private setTime(time: moment.Moment | Date | string): void {
+ const timeMoment = moment.isMoment(time) ? time : moment(time);
+ this.datePickerElement.data('DateTimePicker').date(timeMoment);
}
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.html
index d047d7a..396a277 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.html
@@ -16,16 +16,20 @@
-->
<div [ngClass]="{'dropup': isDropup}">
- <button [ngClass]="['btn', 'btn-link', 'dropdown-toggle', buttonClass]" data-toggle="dropdown">
- <span *ngIf="iconClass || label"
- [ngClass]="{'filter-label': true, 'plain': !isMultipleChoice && !hideCaret && showSelectedValue}">
- <span *ngIf="iconClass" [ngClass]="iconClass"></span>
- <span *ngIf="label">{{label}}</span>
+ <button [ngClass]="['btn', 'dropdown-toggle', buttonClass]" data-toggle="dropdown">
+ <span class="filter-label">
+ <span *ngIf="iconClass || label" [class.plain]="!isMultipleChoice && !hideCaret && showSelectedValue">
+ <span *ngIf="iconClass" [ngClass]="iconClass"></span>
+ <span *ngIf="label && (!selection.length || isMultipleChoice || showCommonLabelWithSelection)"
+ [class.label-before-selection]="isSelectionDisplayable">
+ {{label}}
+ </span>
+ </span>
+ <span *ngIf="isSelectionDisplayable">{{selection[0].label | translate}}</span>
+ <span *ngIf="!hideCaret" class="caret"></span>
</span>
- <span *ngIf="showSelectedValue && !isMultipleChoice && selection.length">{{selection[0].label | translate}}</span>
- <span *ngIf="!hideCaret" class="caret"></span>
</button>
<ul data-component="dropdown-list" [ngClass]="{'dropdown-menu': true, 'dropdown-menu-right': isRightAlign}"
[items]="options" [isMultipleChoice]="isMultipleChoice" (selectedItemChange)="updateSelection($event)"
- [actionArguments]="additionalArgs"></ul>
+ [actionArguments]="listItemArguments"></ul>
</div>
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.less b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.less
index 7b560c1..06861722 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.less
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.less
@@ -25,11 +25,13 @@
text-transform: none;
.filter-label {
- padding: @input-group-addon-padding;
-
- &.plain {
+ .plain {
color: initial;
}
+
+ .label-before-selection {
+ padding: @input-group-addon-padding;
+ }
}
}
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.spec.ts
index b9f9540..8a72c38 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.spec.ts
@@ -16,11 +16,10 @@
* limitations under the License.
*/
-import {NO_ERRORS_SCHEMA, Injector} from '@angular/core';
-import {async, ComponentFixture, TestBed, inject} from '@angular/core/testing';
+import {NO_ERRORS_SCHEMA} from '@angular/core';
+import {async, ComponentFixture, TestBed} from '@angular/core/testing';
import {TranslationModules} from '@app/test-config.spec';
import {StoreModule} from '@ngrx/store';
-import {ServiceInjector} from '@app/classes/service-injector';
import {AppSettingsService, appSettings} from '@app/services/storage/app-settings.service';
import {ClustersService, clusters} from '@app/services/storage/clusters.service';
import {ComponentsService, components} from '@app/services/storage/components.service';
@@ -37,7 +36,6 @@ import {
import {ServiceLogsTruncatedService, serviceLogsTruncated} from '@app/services/storage/service-logs-truncated.service';
import {TabsService, tabs} from '@app/services/storage/tabs.service';
import {UtilsService} from '@app/services/utils.service';
-import {ComponentActionsService} from '@app/services/component-actions.service';
import {HttpClientService} from '@app/services/http-client.service';
import {LogsContainerService} from '@app/services/logs-container.service';
import {AuthService} from '@app/services/auth.service';
@@ -92,7 +90,6 @@ describe('DropdownButtonComponent', () => {
ServiceLogsTruncatedService,
TabsService,
UtilsService,
- ComponentActionsService,
{
provide: HttpClientService,
useValue: httpClient
@@ -105,12 +102,11 @@ describe('DropdownButtonComponent', () => {
.compileComponents();
}));
- beforeEach(inject([Injector], (injector: Injector) => {
- ServiceInjector.injector = injector;
+ beforeEach(() => {
fixture = TestBed.createComponent(DropdownButtonComponent);
component = fixture.componentInstance;
fixture.detectChanges();
- }));
+ });
it('should create component', () => {
expect(component).toBeTruthy();
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.ts
index ead9e1a..b642dd5 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-button/dropdown-button.component.ts
@@ -16,10 +16,8 @@
* limitations under the License.
*/
-import {Component, Input} from '@angular/core';
+import {Component, Input, Output, EventEmitter} from '@angular/core';
import {ListItem} from '@app/classes/list-item';
-import {ServiceInjector} from '@app/classes/service-injector';
-import {ComponentActionsService} from '@app/services/component-actions.service';
import {UtilsService} from '@app/services/utils.service';
@Component({
@@ -30,14 +28,13 @@ import {UtilsService} from '@app/services/utils.service';
export class DropdownButtonComponent {
constructor(protected utils: UtilsService) {
- this.actions = ServiceInjector.injector.get(ComponentActionsService);
}
-
+
@Input()
label?: string;
@Input()
- buttonClass: string = '';
+ buttonClass: string = 'btn-link';
@Input()
iconClass?: string;
@@ -52,10 +49,7 @@ export class DropdownButtonComponent {
options: ListItem[] = [];
@Input()
- action?: string;
-
- @Input()
- additionalArgs: any[] = [];
+ listItemArguments: any[] = [];
@Input()
isMultipleChoice: boolean = false;
@@ -66,7 +60,11 @@ export class DropdownButtonComponent {
@Input()
isDropup: boolean = false;
- private actions: ComponentActionsService;
+ @Input()
+ showCommonLabelWithSelection: boolean = false;
+
+ @Output()
+ selectItem: EventEmitter<any> = new EventEmitter();
protected selectedItems?: ListItem[] = [];
@@ -78,22 +76,28 @@ export class DropdownButtonComponent {
this.selectedItems = items;
}
+ // TODO handle case of selections with multiple items
+ /**
+ * Indicates whether selection can be displayed at the moment, i.e. it's not empty, not multiple
+ * and set to be displayed by showSelectedValue flag
+ * @returns {boolean}
+ */
+ get isSelectionDisplayable():boolean {
+ return this.showSelectedValue && !this.isMultipleChoice && this.selection.length > 0;
+ }
+
updateSelection(item: ListItem): void {
- const action = this.action && this.actions[this.action];
+ const hasAction = this.selectItem.observers.length;
if (this.isMultipleChoice) {
this.options.find((option: ListItem): boolean => {
return this.utils.isEqual(option.value, item.value);
}).isChecked = item.isChecked;
const checkedItems = this.options.filter((option: ListItem): boolean => option.isChecked);
this.selection = checkedItems;
- if (action) {
- action(checkedItems.map((option: ListItem): any => option.value), ...this.additionalArgs);
- }
+ this.selectItem.emit(checkedItems.map((option: ListItem): any => option.value));
} else if (!this.utils.isEqual(this.selection[0], item)) {
this.selection = [item];
- if (action) {
- action(item.value, ...this.additionalArgs);
- }
+ this.selectItem.emit(item.value);
}
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-list/dropdown-list.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-list/dropdown-list.component.html
index 64e4b8e..9861c24 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-list/dropdown-list.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-list/dropdown-list.component.html
@@ -15,19 +15,21 @@
limitations under the License.
-->
-<li *ngFor="let item of items">
- <label class="list-item-label" *ngIf="isMultipleChoice">
- <input type="checkbox" [attr.id]="item.id || item.value" [(ngModel)]="item.isChecked"
- (change)="changeSelectedItem({value: item.value, isChecked: $event.currentTarget.checked})">
- <label [attr.for]="item.id || item.value" class="label-container">
+<li *ngFor="let item of items" [class.divider]="item.isDivider" [attr.role]="item.isDivider ? 'separator' : null">
+ <ng-container *ngIf="!item.isDivider">
+ <label class="list-item-label" *ngIf="isMultipleChoice">
+ <input type="checkbox" [attr.id]="item.id || item.value" [(ngModel)]="item.isChecked"
+ (change)="changeSelectedItem({value: item.value, isChecked: $event.currentTarget.checked})">
+ <label [attr.for]="item.id || item.value" class="label-container">
+ <span *ngIf="item.iconClass" [ngClass]="item.iconClass"></span>
+ {{item.label | translate}}
+ <span #additionalComponent></span>
+ </label>
+ </label>
+ <span class="list-item-label label-container" *ngIf="!isMultipleChoice" (click)="changeSelectedItem(item)">
<span *ngIf="item.iconClass" [ngClass]="item.iconClass"></span>
{{item.label | translate}}
- <div #additionalComponent></div>
- </label>
- </label>
- <span class="list-item-label label-container" *ngIf="!isMultipleChoice" (click)="changeSelectedItem(item)">
- <span *ngIf="item.iconClass" [ngClass]="item.iconClass"></span>
- {{item.label | translate}}
- <div #additionalComponent></div>
- </span>
+ <span #additionalComponent></span>
+ </span>
+ </ng-container>
</li>
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-list/dropdown-list.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-list/dropdown-list.component.spec.ts
index dd602d7..6d88010 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-list/dropdown-list.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-list/dropdown-list.component.spec.ts
@@ -38,8 +38,8 @@ import {TabsService, tabs} from '@app/services/storage/tabs.service';
import {ComponentGeneratorService} from '@app/services/component-generator.service';
import {LogsContainerService} from '@app/services/logs-container.service';
import {HttpClientService} from '@app/services/http-client.service';
-import {ComponentActionsService} from '@app/services/component-actions.service';
import {AuthService} from '@app/services/auth.service';
+import {UtilsService} from '@app/services/utils.service';
import {DropdownListComponent} from './dropdown-list.component';
@@ -84,7 +84,6 @@ describe('DropdownListComponent', () => {
provide: HttpClientService,
useValue: httpClient
},
- ComponentActionsService,
HostsService,
AuditLogsService,
ServiceLogsService,
@@ -98,7 +97,8 @@ describe('DropdownListComponent', () => {
ComponentsService,
ServiceLogsTruncatedService,
TabsService,
- AuthService
+ AuthService,
+ UtilsService
]
})
.compileComponents();
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-list/dropdown-list.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-list/dropdown-list.component.ts
index ef185d0..5d0ad4a 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-list/dropdown-list.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/dropdown-list/dropdown-list.component.ts
@@ -16,33 +16,37 @@
* limitations under the License.
*/
-import {Component, AfterViewInit, Input, Output, EventEmitter, ViewChildren, ViewContainerRef, QueryList} from '@angular/core';
+import {
+ Component, OnChanges, AfterViewChecked, SimpleChanges, Input, Output, EventEmitter, ViewChildren, ViewContainerRef,
+ QueryList
+} from '@angular/core';
import {ListItem} from '@app/classes/list-item';
import {ComponentGeneratorService} from '@app/services/component-generator.service';
-import {ComponentActionsService} from '@app/services/component-actions.service';
@Component({
selector: 'ul[data-component="dropdown-list"]',
templateUrl: './dropdown-list.component.html',
styleUrls: ['./dropdown-list.component.less']
})
-export class DropdownListComponent implements AfterViewInit {
+export class DropdownListComponent implements OnChanges, AfterViewChecked {
- constructor(private componentGenerator: ComponentGeneratorService, private actions: ComponentActionsService) {
+ constructor(private componentGenerator: ComponentGeneratorService) {
}
- ngAfterViewInit() {
- const setter = this.additionalLabelComponentSetter;
- if (setter) {
- this.containers.forEach((container, index) => this.componentGenerator[setter](this.items[index].value, container));
+ private shouldRenderAdditionalComponents: boolean = false;
+
+ ngOnChanges(changes: SimpleChanges): void {
+ if (changes.hasOwnProperty('items')) {
+ this.shouldRenderAdditionalComponents = true;
}
}
- @Input()
- items: ListItem[];
+ ngAfterViewChecked() {
+ this.renderAdditionalComponents();
+ }
@Input()
- defaultAction: Function;
+ items: ListItem[];
@Input()
isMultipleChoice?: boolean = false;
@@ -61,9 +65,18 @@ export class DropdownListComponent implements AfterViewInit {
})
containers: QueryList<ViewContainerRef>;
+ private renderAdditionalComponents(): void {
+ const setter = this.additionalLabelComponentSetter,
+ containers = this.containers;
+ if (this.shouldRenderAdditionalComponents && setter && containers) {
+ containers.forEach((container, index) => this.componentGenerator[setter](this.items[index].value, container));
+ this.shouldRenderAdditionalComponents = false;
+ }
+ }
+
changeSelectedItem(options: ListItem): void {
- if (options.action) {
- this.actions[options.action](...this.actionArguments);
+ if (options.onSelect) {
+ options.onSelect(...this.actionArguments);
}
this.selectedItemChange.emit(options);
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/filter-button/filter-button.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/filter-button/filter-button.component.spec.ts
index 4c93cbe..87c490c 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/filter-button/filter-button.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/filter-button/filter-button.component.spec.ts
@@ -16,11 +16,10 @@
* limitations under the License.
*/
-import {NO_ERRORS_SCHEMA, Injector} from '@angular/core';
-import {async, ComponentFixture, TestBed, inject} from '@angular/core/testing';
+import {NO_ERRORS_SCHEMA} from '@angular/core';
+import {async, ComponentFixture, TestBed} from '@angular/core/testing';
import {TranslationModules} from '@app/test-config.spec';
import {StoreModule} from '@ngrx/store';
-import {ServiceInjector} from '@app/classes/service-injector';
import {AppSettingsService, appSettings} from '@app/services/storage/app-settings.service';
import {ClustersService, clusters} from '@app/services/storage/clusters.service';
import {ComponentsService, components} from '@app/services/storage/components.service';
@@ -36,7 +35,6 @@ import {
} from '@app/services/storage/service-logs-histogram-data.service';
import {ServiceLogsTruncatedService, serviceLogsTruncated} from '@app/services/storage/service-logs-truncated.service';
import {TabsService, tabs} from '@app/services/storage/tabs.service';
-import {ComponentActionsService} from '@app/services/component-actions.service';
import {UtilsService} from '@app/services/utils.service';
import {HttpClientService} from '@app/services/http-client.service';
import {LogsContainerService} from '@app/services/logs-container.service';
@@ -91,7 +89,6 @@ describe('FilterButtonComponent', () => {
ServiceLogsHistogramDataService,
ServiceLogsTruncatedService,
TabsService,
- ComponentActionsService,
UtilsService,
{
provide: HttpClientService,
@@ -105,12 +102,11 @@ describe('FilterButtonComponent', () => {
.compileComponents();
}));
- beforeEach(inject([Injector], (injector: Injector) => {
- ServiceInjector.injector = injector;
+ beforeEach(() => {
fixture = TestBed.createComponent(FilterButtonComponent);
component = fixture.componentInstance;
fixture.detectChanges();
- }));
+ });
it('should create component', () => {
expect(component).toBeTruthy();
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/filter-dropdown/filter-dropdown.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/filter-dropdown/filter-dropdown.component.spec.ts
index 61d30d1..e3105c1 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/filter-dropdown/filter-dropdown.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/filter-dropdown/filter-dropdown.component.spec.ts
@@ -15,11 +15,10 @@
* limitations under the License.
*/
-import {NO_ERRORS_SCHEMA, Injector} from '@angular/core';
-import {async, ComponentFixture, TestBed, inject} from '@angular/core/testing';
+import {NO_ERRORS_SCHEMA} from '@angular/core';
+import {async, ComponentFixture, TestBed} from '@angular/core/testing';
import {TranslationModules} from '@app/test-config.spec';
import {StoreModule} from '@ngrx/store';
-import {ServiceInjector} from '@app/classes/service-injector';
import {AppSettingsService, appSettings} from '@app/services/storage/app-settings.service';
import {AppStateService, appState} from '@app/services/storage/app-state.service';
import {AuditLogsService, auditLogs} from '@app/services/storage/audit-logs.service';
@@ -36,7 +35,6 @@ import {ClustersService, clusters} from '@app/services/storage/clusters.service'
import {ComponentsService, components} from '@app/services/storage/components.service';
import {HostsService, hosts} from '@app/services/storage/hosts.service';
import {UtilsService} from '@app/services/utils.service';
-import {ComponentActionsService} from '@app/services/component-actions.service';
import {LogsContainerService} from '@app/services/logs-container.service';
import {HttpClientService} from '@app/services/http-client.service';
import {AuthService} from '@app/services/auth.service';
@@ -111,7 +109,6 @@ describe('FilterDropdownComponent', () => {
useValue: filtering
},
UtilsService,
- ComponentActionsService,
LogsContainerService,
{
provide: HttpClientService,
@@ -124,12 +121,11 @@ describe('FilterDropdownComponent', () => {
.compileComponents();
}));
- beforeEach(inject([Injector], (injector: Injector) => {
- ServiceInjector.injector = injector;
+ beforeEach(() => {
fixture = TestBed.createComponent(FilterDropdownComponent);
component = fixture.componentInstance;
fixture.detectChanges();
- }));
+ });
it('should create component', () => {
expect(component).toBeTruthy();
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/filter-dropdown/filter-dropdown.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/filter-dropdown/filter-dropdown.component.ts
index 665386b..b85c572 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/filter-dropdown/filter-dropdown.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/filter-dropdown/filter-dropdown.component.ts
@@ -17,7 +17,6 @@
import {Component, forwardRef} from '@angular/core';
import {ControlValueAccessor, NG_VALUE_ACCESSOR} from '@angular/forms';
-import {UtilsService} from '@app/services/utils.service';
import {DropdownButtonComponent} from '@app/components/dropdown-button/dropdown-button.component';
import {ListItem} from '@app/classes/list-item';
@@ -35,10 +34,6 @@ import {ListItem} from '@app/classes/list-item';
})
export class FilterDropdownComponent extends DropdownButtonComponent implements ControlValueAccessor {
- constructor(protected utils: UtilsService) {
- super(utils);
- }
-
private onChange: (fn: any) => void;
get selection(): ListItem[] {
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.html
index f0cf3f4..9b0ac41 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.html
@@ -29,13 +29,16 @@
</button>
</div>
<div class="filter-buttons col-md-4">
- <dropdown-button [options]="searchBoxItems | async" iconClass="fa fa-search-minus" action="proceedWithExclude"
- label="{{'filter.exclude' | translate}}" [hideCaret]="true"
- [showSelectedValue]="false"></dropdown-button>
+ <dropdown-button iconClass="fa fa-search-minus" label="{{'filter.exclude' | translate}}" [hideCaret]="true"
+ [showSelectedValue]="false" [showCommonLabelWithSelection]="true"
+ [options]="searchBoxItems | async" (selectItem)="proceedWithExclude($event)"></dropdown-button>
<filter-button *ngIf="isFilterConditionDisplayed('hosts')" formControlName="hosts"
label="{{filters.hosts.label | translate}}" [iconClass]="filters.hosts.iconClass"
[subItems]="filters.hosts.options" [isMultipleChoice]="true" [isRightAlign]="true"
additionalLabelComponentSetter="getDataForHostsNodeBar"></filter-button>
+ <filter-button *ngIf="isFilterConditionDisplayed('users')" formControlName="users"
+ label="{{filters.users.label | translate}}" [iconClass]="filters.users.iconClass"
+ [subItems]="filters.users.options" [isMultipleChoice]="true" [isRightAlign]="true"></filter-button>
<filter-button *ngIf="isFilterConditionDisplayed('components')" formControlName="components"
label="{{filters.components.label | translate}}" [iconClass]="filters.components.iconClass"
[subItems]="filters.components.options" [isMultipleChoice]="true" [isRightAlign]="true"
@@ -43,9 +46,5 @@
<filter-button *ngIf="isFilterConditionDisplayed('levels')" formControlName="levels"
label="{{filters.levels.label | translate}}" [iconClass]="filters.levels.iconClass"
[subItems]="filters.levels.options" [isMultipleChoice]="true" [isRightAlign]="true"></filter-button>
- <menu-button *ngIf="!captureSeconds" label="{{'filter.capture' | translate}}" iconClass="fa fa-caret-right"
- action="startCapture"></menu-button>
- <menu-button *ngIf="captureSeconds" label="{{captureSeconds | timerSeconds}}" iconClass="fa fa-stop stop-icon"
- action="stopCapture"></menu-button>
</div>
</form>
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.less b/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.less
index e2f9a1e..94bd1db 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.less
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.less
@@ -68,10 +68,6 @@
}
}
- /deep/ .stop-icon {
- color: @exclude-color;
- }
-
.filter-buttons {
.default-flex;
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.spec.ts
index 8df1d54..29cac93 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.spec.ts
@@ -39,7 +39,6 @@ import {TabsService, tabs} from '@app/services/storage/tabs.service';
import {HttpClientService} from '@app/services/http-client.service';
import {UtilsService} from '@app/services/utils.service';
import {LogsContainerService} from '@app/services/logs-container.service';
-import {TimerSecondsPipe} from '@app/pipes/timer-seconds.pipe';
import {FiltersPanelComponent} from './filters-panel.component';
@@ -58,8 +57,7 @@ describe('FiltersPanelComponent', () => {
};
TestBed.configureTestingModule({
declarations: [
- FiltersPanelComponent,
- TimerSecondsPipe
+ FiltersPanelComponent
],
imports: [
StoreModule.provideStore({
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.ts
index cd372ec..b233847 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/filters-panel/filters-panel.component.ts
@@ -95,10 +95,6 @@ export class FiltersPanelComponent implements OnChanges {
return this.logsContainer.queryParameterAdd;
}
- get captureSeconds(): number {
- return this.logsContainer.captureSeconds;
- }
-
searchBoxValueUpdate: Subject<void> = new Subject();
isFilterConditionDisplayed(key: string): boolean {
@@ -109,4 +105,13 @@ export class FiltersPanelComponent implements OnChanges {
this.searchBoxValueUpdate.next();
}
+ proceedWithExclude(item: string): void {
+ this.queryParameterNameChange.next({
+ item: {
+ value: item
+ },
+ isExclude: true
+ });
+ }
+
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/history-item-controls/history-item-controls.component.html
similarity index 71%
copy from ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.html
copy to ambari-logsearch/ambari-logsearch-web/src/app/components/history-item-controls/history-item-controls.component.html
index ab6326a..e6978f1 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/history-item-controls/history-item-controls.component.html
@@ -14,7 +14,7 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<menu-button *ngFor="let item of items" label="{{item.label | translate}}" [action]="item.action"
- [iconClass]="item.iconClass" [labelClass]="item.labelClass" [subItems]="item.subItems"
- [hideCaret]="item.hideCaret" [badge]="item.badge" [isRightAlign]="item.isRightAlign">
-</menu-button>
+
+<!-- TODO implement View details and Save filter actions -->
+<span class="fa fa-info-circle"></span>
+<span class="fa fa-floppy-o"></span>
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/object.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/history-item-controls/history-item-controls.component.less
similarity index 93%
copy from ambari-logsearch/ambari-logsearch-web/src/app/classes/object.ts
copy to ambari-logsearch/ambari-logsearch-web/src/app/components/history-item-controls/history-item-controls.component.less
index 4d0c7f6..dfb9997 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/object.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/history-item-controls/history-item-controls.component.less
@@ -16,4 +16,6 @@
* limitations under the License.
*/
-export type HomogeneousObject<T> = {[key: string]: T};
+:host {
+ float: right;
+}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/history-item-controls/history-item-controls.component.spec.ts
similarity index 70%
copy from ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.spec.ts
copy to ambari-logsearch/ambari-logsearch-web/src/app/components/history-item-controls/history-item-controls.component.spec.ts
index 081304e..4dbaa2d 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/history-item-controls/history-item-controls.component.spec.ts
@@ -16,27 +16,23 @@
* limitations under the License.
*/
-import {CUSTOM_ELEMENTS_SCHEMA} from '@angular/core';
import {async, ComponentFixture, TestBed} from '@angular/core/testing';
-import {TranslationModules} from '@app/test-config.spec';
-import {ActionMenuComponent} from './action-menu.component';
+import {HistoryItemControlsComponent} from './history-item-controls.component';
-describe('ActionMenuComponent', () => {
- let component: ActionMenuComponent;
- let fixture: ComponentFixture<ActionMenuComponent>;
+describe('HistoryItemControlsComponent', () => {
+ let component: HistoryItemControlsComponent;
+ let fixture: ComponentFixture<HistoryItemControlsComponent>;
beforeEach(async(() => {
TestBed.configureTestingModule({
- imports: TranslationModules,
- declarations: [ActionMenuComponent],
- schemas: [CUSTOM_ELEMENTS_SCHEMA]
+ declarations: [HistoryItemControlsComponent]
})
.compileComponents();
}));
beforeEach(() => {
- fixture = TestBed.createComponent(ActionMenuComponent);
+ fixture = TestBed.createComponent(HistoryItemControlsComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-top-resources-query-params.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/history-item-controls/history-item-controls.component.ts
similarity index 72%
copy from ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-top-resources-query-params.ts
copy to ambari-logsearch/ambari-logsearch-web/src/app/components/history-item-controls/history-item-controls.component.ts
index 0d12539..1975d9a 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/classes/queries/audit-logs-top-resources-query-params.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/history-item-controls/history-item-controls.component.ts
@@ -16,8 +16,13 @@
* limitations under the License.
*/
-import {AuditLogsQueryParams} from '@app/classes/queries/audit-logs-query-params';
+import {Component} from '@angular/core';
-export class AuditLogsTopResourcesQueryParams extends AuditLogsQueryParams {
- field: string;
+@Component({
+ selector: 'history-item-controls',
+ templateUrl: './history-item-controls.component.html',
+ styleUrls: ['./history-item-controls.component.less']
+})
+export class HistoryItemControlsComponent {
+ // TODO implement View details and Save filter actions
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/log-context/log-context.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/log-context/log-context.component.spec.ts
index c346c9a..6e6a70e 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/log-context/log-context.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/log-context/log-context.component.spec.ts
@@ -37,6 +37,7 @@ import {TranslationModules} from '@app/test-config.spec';
import {ModalComponent} from '@app/components/modal/modal.component';
import {LogsContainerService} from '@app/services/logs-container.service';
import {HttpClientService} from '@app/services/http-client.service';
+import {UtilsService} from '@app/services/utils.service';
import {LogContextComponent} from './log-context.component';
@@ -94,7 +95,8 @@ describe('LogContextComponent', () => {
{
provide: HttpClientService,
useValue: httpClient
- }
+ },
+ UtilsService
],
schemas: [CUSTOM_ELEMENTS_SCHEMA]
})
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/log-index-filter/log-index-filter.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/log-index-filter/log-index-filter.component.html
new file mode 100644
index 0000000..50d7742
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/log-index-filter/log-index-filter.component.html
@@ -0,0 +1,86 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<div>{{'logIndexFilter.caption' | translate}}</div>
+<h5>{{'logIndexFilter.selectCluster' | translate}}</h5>
+<dropdown-button [options]="clustersListItems | async" (selectItem)="setActiveCluster($event)"
+ label="{{'logIndexFilter.select' | translate}}" buttonClass="btn-default"></dropdown-button>
+<table *ngIf="activeClusterName" class="table table-hover">
+ <thead>
+ <tr>
+ <th class="component-column">{{'filter.components' | translate}}</th>
+ <th *ngFor="let column of columns" class="checkbox-column">
+ <input type="checkbox" attr.id="{{column.name}}"
+ [attr.checked]="isAllComponentsCheckedForLevel(column.name) ? 'checked' : null"
+ (change)="processAllComponentsForLevel(column.name, $event.target.checked)">
+ <label attr.for="{{column.name}}">
+ <graph-legend-item label="{{column.label | translate}}" color="{{column.color}}"></graph-legend-item>
+ </label>
+ </th>
+ <th class="override-column">{{'logIndexFilter.override' | translate}}</th>
+ </tr>
+ </thead>
+ <tbody>
+ <ng-container *ngFor="let component of activeClusterConfigs">
+ <tr>
+ <td class="component-column">
+ <input type="checkbox" attr.id="{{component.name}}"
+ [attr.checked]="isAllLevelsCheckedForComponent(component.name) ? 'checked' : null"
+ (change)="processAllLevelsForComponent(component.name, $event.target.checked)">
+ <label attr.for="{{component.name}}">{{component.label}}</label>
+ </td>
+ <td *ngFor="let levelName of levelNames" class="checkbox-column">
+ <input type="checkbox" attr.id="{{getCheckBoxId(component.name, levelName)}}"
+ [(ngModel)]="component[levelName].defaults" (change)="updateValue()">
+ <label attr.for="{{getCheckBoxId(component.name, levelName)}}"> </label>
+ </td>
+ <td class="override-column">
+ <span class="overrides-toggle">{{'logIndexFilter.addHosts' | translate}}</span>
+ </td>
+ </tr>
+ <tr> <!--overrides row -->
+ <td class="component-column">
+ <input type="checkbox" attr.id="{{component.name}}_overrides"
+ [attr.checked]="isAllLevelsCheckedForComponent(component.name, true) ? 'checked' : null"
+ (change)="processAllLevelsForComponent(component.name, $event.target.checked, true)">
+ <label attr.for="{{component.name}}_overrides"> </label>
+ </td>
+ <td *ngFor="let levelName of levelNames" class="checkbox-column">
+ <input type="checkbox" attr.id="{{getCheckBoxId(component.name, levelName, true)}}"
+ [(ngModel)]="component[levelName].overrides" (change)="updateValue()">
+ <label attr.for="{{getCheckBoxId(component.name, levelName, true)}}"> </label>
+ </td>
+ <td class="override-column">
+ <div class="col-md-12 row">
+ <div class="col-md-6">
+ <div class="text-uppercase">{{'logIndexFilter.hostname' | translate}}</div>
+ <input type="text" class="form-control" [(ngModel)]="component.hosts" (change)="updateValue()">
+ <!-- TODO implement typeahead similar to search box
+ (some functionality and appearance most likely will be shared) -->
+ <!-- <input type="text" class="form-control" [(ngModel)]="component.hosts" (change)="updateValue()"
+ [typeahead]="hosts | async" typeaheadOptionField="name"> -->
+ </div>
+ <div class=" col-md-6">
+ <div class="text-uppercase">{{'logIndexFilter.expiryDate' | translate}}</div>
+ <date-picker [time]="component.expiryTime" (timeChange)="setExpiryTime($event, component)"></date-picker>
+ </div>
+ </div>
+ </td>
+ </tr>
+ </ng-container>
+ </tbody>
+</table>
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/services/storage/filters.service.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/log-index-filter/log-index-filter.component.less
similarity index 62%
rename from ambari-logsearch/ambari-logsearch-web/src/app/services/storage/filters.service.ts
rename to ambari-logsearch/ambari-logsearch-web/src/app/components/log-index-filter/log-index-filter.component.less
index 493e2e6..f932258 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/services/storage/filters.service.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/log-index-filter/log-index-filter.component.less
@@ -16,18 +16,38 @@
* limitations under the License.
*/
+@import '../mixins';
-import {Injectable} from '@angular/core';
-import {Store} from '@ngrx/store';
-import {AppStore, CollectionModelService, getCollectionReducer} from '@app/classes/models/store';
+table {
+ display: block;
+ max-height: 70vh;
+ overflow-y: auto;
+ table-layout: fixed;
-export const modelName = 'filters';
+ .component-column {
+ width: auto;
+ }
+
+ .checkbox-column {
+ width: 100px;
+ padding-left: 0;
+ padding-right: 0;
-@Injectable()
-export class FiltersService extends CollectionModelService {
- constructor(store: Store<AppStore>) {
- super(modelName, store);
+ /deep/ graph-legend-item {
+ padding-right: 1px;
+ }
+ }
+
+ .override-column {
+ width: 32%;
+ padding-right: 0;
}
-}
-export const filters = getCollectionReducer(modelName);
+ .overrides-toggle {
+ .clickable-item;
+ }
+
+ input[type=checkbox] + label {
+ font-size: @table-font-size;
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/context-menu/context-menu.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/log-index-filter/log-index-filter.component.spec.ts
similarity index 83%
copy from ambari-logsearch/ambari-logsearch-web/src/app/components/context-menu/context-menu.component.spec.ts
copy to ambari-logsearch/ambari-logsearch-web/src/app/components/log-index-filter/log-index-filter.component.spec.ts
index 1c5154b..a236686 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/context-menu/context-menu.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/log-index-filter/log-index-filter.component.spec.ts
@@ -19,9 +19,8 @@
import {CUSTOM_ELEMENTS_SCHEMA} from '@angular/core';
import {async, ComponentFixture, TestBed} from '@angular/core/testing';
import {FormsModule} from '@angular/forms';
-import {StoreModule} from '@ngrx/store';
import {TranslationModules} from '@app/test-config.spec';
-import {HostsService, hosts} from '@app/services/storage/hosts.service';
+import {StoreModule} from '@ngrx/store';
import {AuditLogsService, auditLogs} from '@app/services/storage/audit-logs.service';
import {ServiceLogsService, serviceLogs} from '@app/services/storage/service-logs.service';
import {AuditLogsFieldsService, auditLogsFields} from '@app/services/storage/audit-logs-fields.service';
@@ -34,40 +33,37 @@ import {AppSettingsService, appSettings} from '@app/services/storage/app-setting
import {AppStateService, appState} from '@app/services/storage/app-state.service';
import {ClustersService, clusters} from '@app/services/storage/clusters.service';
import {ComponentsService, components} from '@app/services/storage/components.service';
+import {HostsService, hosts} from '@app/services/storage/hosts.service';
import {ServiceLogsTruncatedService, serviceLogsTruncated} from '@app/services/storage/service-logs-truncated.service';
import {TabsService, tabs} from '@app/services/storage/tabs.service';
import {ComponentGeneratorService} from '@app/services/component-generator.service';
-import {LogsContainerService} from '@app/services/logs-container.service';
import {HttpClientService} from '@app/services/http-client.service';
-import {ComponentActionsService} from '@app/services/component-actions.service';
-import {AuthService} from '@app/services/auth.service';
+import {LogsContainerService} from '@app/services/logs-container.service';
+import {UserSettingsService} from '@app/services/user-settings.service';
+import {UtilsService} from '@app/services/utils.service';
+import {DropdownButtonComponent} from '@app/components/dropdown-button/dropdown-button.component';
import {DropdownListComponent} from '@app/components/dropdown-list/dropdown-list.component';
-import {ContextMenuComponent} from './context-menu.component';
+import {LogIndexFilterComponent} from './log-index-filter.component';
-describe('ContextMenuComponent', () => {
- let component: ContextMenuComponent;
- let fixture: ComponentFixture<ContextMenuComponent>;
+describe('LogIndexFilterComponent', () => {
+ let component: LogIndexFilterComponent;
+ let fixture: ComponentFixture<LogIndexFilterComponent>;
- const httpClient = {
- get: () => {
- return {
- subscribe: () => {
+ beforeEach(async(() => {
+ const httpClient = {
+ get: () => {
+ return {
+ subscribe: () => {
+ }
}
}
- }
- };
-
- beforeEach(async(() => {
+ };
TestBed.configureTestingModule({
- declarations: [
- ContextMenuComponent,
- DropdownListComponent
- ],
imports: [
+ FormsModule,
...TranslationModules,
StoreModule.provideStore({
- hosts,
auditLogs,
serviceLogs,
auditLogsFields,
@@ -78,20 +74,25 @@ describe('ContextMenuComponent', () => {
appState,
clusters,
components,
+ hosts,
serviceLogsTruncated,
tabs
- }),
- FormsModule
+ })
+ ],
+ declarations: [
+ LogIndexFilterComponent,
+ DropdownButtonComponent,
+ DropdownListComponent
],
providers: [
ComponentGeneratorService,
- LogsContainerService,
{
provide: HttpClientService,
useValue: httpClient
},
- ComponentActionsService,
- HostsService,
+ LogsContainerService,
+ UserSettingsService,
+ UtilsService,
AuditLogsService,
ServiceLogsService,
AuditLogsFieldsService,
@@ -102,9 +103,9 @@ describe('ContextMenuComponent', () => {
AppStateService,
ClustersService,
ComponentsService,
+ HostsService,
ServiceLogsTruncatedService,
- TabsService,
- AuthService
+ TabsService
],
schemas: [CUSTOM_ELEMENTS_SCHEMA]
})
@@ -112,7 +113,7 @@ describe('ContextMenuComponent', () => {
}));
beforeEach(() => {
- fixture = TestBed.createComponent(ContextMenuComponent);
+ fixture = TestBed.createComponent(LogIndexFilterComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/log-index-filter/log-index-filter.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/log-index-filter/log-index-filter.component.ts
new file mode 100644
index 0000000..a65555f
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/log-index-filter/log-index-filter.component.ts
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Component, OnInit, Output, EventEmitter, forwardRef} from '@angular/core';
+import {ControlValueAccessor, NG_VALUE_ACCESSOR} from '@angular/forms';
+import {Observable} from 'rxjs/Observable';
+import 'rxjs/add/operator/map';
+import {Moment} from 'moment';
+import {ListItem} from '@app/classes/list-item';
+import {HomogeneousObject, LogLevelObject} from '@app/classes/object';
+import {LogIndexFilterComponentConfig} from '@app/classes/settings';
+import {LogLevel} from '@app/classes/string';
+import {LogsContainerService} from '@app/services/logs-container.service';
+import {UserSettingsService} from '@app/services/user-settings.service';
+import {UtilsService} from '@app/services/utils.service';
+import {ClustersService} from '@app/services/storage/clusters.service';
+import {HostsService} from '@app/services/storage/hosts.service';
+
+@Component({
+ selector: 'log-index-filter',
+ templateUrl: './log-index-filter.component.html',
+ styleUrls: ['./log-index-filter.component.less'],
+ providers: [
+ {
+ provide: NG_VALUE_ACCESSOR,
+ useExisting: forwardRef(() => LogIndexFilterComponent),
+ multi: true
+ }
+ ]
+})
+export class LogIndexFilterComponent implements OnInit, ControlValueAccessor {
+
+ constructor(
+ private logsContainer: LogsContainerService, private settingsService: UserSettingsService,
+ private utils: UtilsService, private clustersStorage: ClustersService, private hostsStorage: HostsService
+ ) {
+ }
+
+ ngOnInit() {
+ this.changeIsSubmitDisabled.emit(true);
+ this.clusters.subscribe((clusters: string[]) => this.settingsService.loadIndexFilterConfig(clusters));
+ }
+
+ @Output()
+ changeIsSubmitDisabled: EventEmitter<boolean> = new EventEmitter();
+
+ private onChange: (fn: any) => void;
+
+ readonly columns: LogLevelObject[] = this.logsContainer.logLevels;
+
+ readonly levelNames: LogLevel[] = this.columns.map((level: LogLevelObject): LogLevel => level.name);
+
+ clusters: Observable<string[]> = this.clustersStorage.getAll();
+
+ hosts: Observable<string[]> = this.hostsStorage.getAll();
+
+ clustersListItems: Observable<ListItem[]> = this.clusters.map((clusterNames: string[]): ListItem[] => {
+ return clusterNames.map(this.utils.getListItemFromString);
+ });
+
+ activeClusterName: string = '';
+
+ /**
+ * Configs for all clusters
+ */
+ private configs: HomogeneousObject<LogIndexFilterComponentConfig[]>;
+
+ /**
+ * Configs for selected cluster
+ * @returns {LogIndexFilterComponentConfig[]}
+ */
+ get activeClusterConfigs(): LogIndexFilterComponentConfig[] {
+ return this.configs[this.activeClusterName];
+ }
+
+ /**
+ * Select or unselect checkboxes for all log levels for given component
+ * @param {string} componentName
+ * @param {boolean} isChecked
+ * @param {boolean} isOverride - indicates whether levels for override are processed
+ */
+ processAllLevelsForComponent(componentName: string, isChecked: boolean, isOverride: boolean = false): void {
+ const componentConfig = this.getComponentConfigs(componentName),
+ key = isOverride ? 'overrides' : 'defaults';
+ this.levelNames.forEach((levelName: LogLevel) => componentConfig[levelName][key] = isChecked);
+ this.updateValue();
+ }
+
+ /**
+ * Select or unselect checkboxes for all components for given log level
+ * @param {LogLevel} levelName
+ * @param {boolean} isChecked
+ */
+ processAllComponentsForLevel(levelName: LogLevel, isChecked: boolean): void {
+ this.activeClusterConfigs.forEach((component: LogIndexFilterComponentConfig): void => {
+ component[levelName].defaults = isChecked;
+ component[levelName].overrides = isChecked;
+ });
+ this.updateValue();
+ }
+
+ /**
+ * Indicates whether all log levels for given component are checked
+ * @param {string} componentName
+ * @param {string} isOverride - indicates whether levels for override are overviewed
+ * @returns {boolean}
+ */
+ isAllLevelsCheckedForComponent(componentName: string, isOverride: boolean = false): boolean {
+ const componentConfig = this.getComponentConfigs(componentName),
+ key = isOverride ? 'overrides' : 'defaults';
+ return this.levelNames.every((levelName: LogLevel): boolean => componentConfig[levelName][key]);
+ }
+
+ /**
+ * Indicates whether all components for given log level are checked
+ * @param {LogLevel} levelName
+ * @returns {boolean}
+ */
+ isAllComponentsCheckedForLevel(levelName: LogLevel): boolean {
+ return this.activeClusterConfigs.every((component: LogIndexFilterComponentConfig): boolean => {
+ return component[levelName].defaults;
+ });
+ }
+
+ setActiveCluster(clusterName: string): void {
+ this.activeClusterName = clusterName;
+ this.changeIsSubmitDisabled.emit(false);
+ }
+
+ getCheckBoxId(componentName: string, levelName: string, isOverride: boolean = false): string {
+ return `component_${componentName}_level_${levelName}${isOverride ? '_override' : ''}`;
+ }
+
+ setExpiryTime(time: Moment, componentConfig): void {
+ componentConfig.expiryTime = time.toISOString();
+ }
+
+ private getComponentConfigs(componentName: string) {
+ return this.activeClusterConfigs.find((component: LogIndexFilterComponentConfig): boolean => {
+ return component.name === componentName;
+ });
+ }
+
+ writeValue(filters: HomogeneousObject<LogIndexFilterComponentConfig[]>): void {
+ this.configs = filters;
+ this.updateValue();
+ }
+
+ registerOnChange(callback: any): void {
+ this.onChange = callback;
+ }
+
+ registerOnTouched(): void {
+ }
+
+ updateValue(): void {
+ if(this.onChange) {
+ this.onChange(this.configs);
+ }
+ }
+
+}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.ts
index d853bb5..e8ccc08 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/logs-container/logs-container.component.ts
@@ -30,8 +30,8 @@ import {Tab} from '@app/classes/models/tab';
import {BarGraph} from '@app/classes/models/bar-graph';
import {ActiveServiceLogEntry} from '@app/classes/active-service-log-entry';
import {ListItem} from '@app/classes/list-item';
-import {HomogeneousObject} from '@app/classes/object';
-import {LogsType} from '@app/classes/string';
+import {HomogeneousObject, LogLevelObject} from '@app/classes/object';
+import {LogsType, LogLevel} from '@app/classes/string';
import {FiltersPanelComponent} from "@app/components/filters-panel/filters-panel.component";
@Component({
@@ -52,7 +52,11 @@ export class LogsContainerComponent implements OnInit {
this.logsContainer.loadColumnsNames();
this.appState.getParameter('activeLogsType').subscribe((value: LogsType) => this.logsType = value);
this.serviceLogsHistogramStorage.getAll().subscribe((data: BarGraph[]): void => {
- this.serviceLogsHistogramData = this.logsContainer.getGraphData(data, Object.keys(this.logsContainer.colors));
+ this.serviceLogsHistogramData = this.logsContainer.getGraphData(data, this.logsContainer.logLevels.map((
+ level: LogLevelObject
+ ): LogLevel => {
+ return level.name;
+ }));
});
this.auditLogsGraphStorage.getAll().subscribe((data: BarGraph[]): void => {
this.auditLogsGraphData = this.logsContainer.getGraphData(data);
@@ -88,9 +92,13 @@ export class LogsContainerComponent implements OnInit {
auditLogsGraphData: HomogeneousObject<HomogeneousObject<number>>;
- get serviceLogsHistogramColors(): HomogeneousObject<string> {
- return this.logsContainer.colors;
- }
+ serviceLogsHistogramColors: HomogeneousObject<string> = this.logsContainer.logLevels.reduce((
+ currentObject: HomogeneousObject<string>, level: LogLevelObject
+ ): HomogeneousObject<string> => {
+ return Object.assign({}, currentObject, {
+ [level.name]: level.color
+ });
+ }, {});
get autoRefreshRemainingSeconds(): number {
return this.logsContainer.autoRefreshRemainingSeconds;
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.html
index 5e2b15f..8e67b5d 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.html
@@ -23,7 +23,9 @@
<i *ngIf="hasCaret" [ngClass]="['fa ', caretClass ]"></i>
<span *ngIf="label" class="menu-button-label">{{label}}</span>
</a>
- <ul data-component="dropdown-list" *ngIf="hasSubItems" [items]="subItems" (selectedItemChange)="onDropdownItemChange($event)"
- [isMultipleChoice]="isMultipleChoice" [additionalLabelComponentSetter]="additionalLabelComponentSetter"
- [ngClass]="{'dropdown-menu': true, 'dropdown-menu-right': isRightAlign}"></ul>
+ <ul data-component="dropdown-list" *ngIf="hasSubItems" [items]="subItems"
+ (selectedItemChange)="onDropdownItemChange($event)" [isMultipleChoice]="isMultipleChoice"
+ [additionalLabelComponentSetter]="additionalLabelComponentSetter"
+ [ngClass]="'dropdown-menu' + (isRightAlign ? ' dropdown-menu-right' : '') + (listClass ? ' ' + listClass : '')"
+ ></ul>
</div>
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.spec.ts
index 3b210a3..4e77db5 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.spec.ts
@@ -16,10 +16,9 @@
* limitations under the License.
*/
-import {NO_ERRORS_SCHEMA, Injector} from '@angular/core';
-import {async, ComponentFixture, TestBed, inject} from '@angular/core/testing';
+import {NO_ERRORS_SCHEMA} from '@angular/core';
+import {async, ComponentFixture, TestBed} from '@angular/core/testing';
import {TranslationModules} from '@app/test-config.spec';
-import {ServiceInjector} from '@app/classes/service-injector';
import {StoreModule} from '@ngrx/store';
import {AppSettingsService, appSettings} from '@app/services/storage/app-settings.service';
import {AppStateService, appState} from '@app/services/storage/app-state.service';
@@ -36,7 +35,6 @@ import {
} from '@app/services/storage/service-logs-histogram-data.service';
import {ServiceLogsTruncatedService, serviceLogsTruncated} from '@app/services/storage/service-logs-truncated.service';
import {TabsService, tabs} from '@app/services/storage/tabs.service';
-import {ComponentActionsService} from '@app/services/component-actions.service';
import {HttpClientService} from '@app/services/http-client.service';
import {LogsContainerService} from '@app/services/logs-container.service';
import {AuthService} from '@app/services/auth.service';
@@ -90,7 +88,6 @@ describe('MenuButtonComponent', () => {
ServiceLogsHistogramDataService,
ServiceLogsTruncatedService,
TabsService,
- ComponentActionsService,
{
provide: HttpClientService,
useValue: httpClient
@@ -103,12 +100,11 @@ describe('MenuButtonComponent', () => {
.compileComponents();
}));
- beforeEach(inject([Injector], (injector: Injector) => {
- ServiceInjector.injector = injector;
+ beforeEach(() => {
fixture = TestBed.createComponent(MenuButtonComponent);
component = fixture.componentInstance;
fixture.detectChanges();
- }));
+ });
it('should create component', () => {
expect(component).toBeTruthy();
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.ts
index 12da4ac..432b561 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.ts
@@ -16,10 +16,8 @@
* limitations under the License.
*/
-import {Component, Input, ViewChild, ElementRef} from '@angular/core';
+import {Component, Input, Output, ViewChild, ElementRef, EventEmitter} from '@angular/core';
import {ListItem} from '@app/classes/list-item';
-import {ServiceInjector} from '@app/classes/service-injector';
-import {ComponentActionsService} from '@app/services/component-actions.service';
@Component({
selector: 'menu-button',
@@ -28,10 +26,6 @@ import {ComponentActionsService} from '@app/services/component-actions.service';
})
export class MenuButtonComponent {
- constructor() {
- this.actions = ServiceInjector.injector.get(ComponentActionsService);
- }
-
@ViewChild('dropdown')
dropdown: ElementRef;
@@ -39,9 +33,6 @@ export class MenuButtonComponent {
label?: string;
@Input()
- action: string;
-
- @Input()
iconClass: string;
@Input()
@@ -84,7 +75,14 @@ export class MenuButtonComponent {
@Input()
maxLongClickDelay: number = 0;
- private actions: ComponentActionsService;
+ @Input()
+ listClass: string = '';
+
+ @Output()
+ buttonClick: EventEmitter<void> = new EventEmitter();
+
+ @Output()
+ selectItem: EventEmitter<ListItem> = new EventEmitter();
/**
* This is a private property to indicate the mousedown timestamp, so that we can check it when teh click event
@@ -122,14 +120,14 @@ export class MenuButtonComponent {
!this.maxLongClickDelay || mdt + this.maxLongClickDelay >= now
);
let openDropdown = this.hasSubItems && (
- el.classList.contains(this.caretClass) || isLongClick || !this.actions[this.action]
+ el.classList.contains(this.caretClass) || isLongClick || !this.buttonClick.observers.length
);
if (openDropdown && this.dropdown) {
if (this.toggleDropdown()) {
this.listenToClickOut();
}
- } else if (this.action) {
- this.actions[this.action]();
+ } else if (this.buttonClick.observers.length) {
+ this.buttonClick.emit();
}
this.mouseDownTimestamp = 0;
event.preventDefault();
@@ -211,7 +209,7 @@ export class MenuButtonComponent {
}
updateSelection(options: ListItem) {
- // TODO implement value change behaviour
+ this.selectItem.emit(options);
}
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/modal/modal.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/modal/modal.component.html
index 398a429..799b840 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/modal/modal.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/modal/modal.component.html
@@ -17,7 +17,9 @@
<div class="modal-backdrop in"></div>
<div class="modal in">
- <div [ngClass]="{'modal-dialog': true, 'modal-sm': isSmallModal, 'modal-lg': isLargeModal}">
+ <div [ngClass]="{
+ 'modal-dialog': true, 'modal-sm': isSmallModal, 'modal-lg': isLargeModal, 'modal-xl': isExtraLargeModal
+ }">
<div class="modal-content">
<div *ngIf="showHeader" class="modal-header">
<button *ngIf="showCloseButton" type="button" class="close" data-dismiss="modal" (click)="onClose()">
@@ -33,6 +35,7 @@
<button *ngIf="showCancelButton" class="btn {{cancelButtonClassName}}"
(click)="onCancel()">{{cancelButtonLabel | translate}}</button>
<button *ngIf="showSubmitButton" class="btn {{submitButtonClassName}}"
+ [attr.disabled]="isSubmitDisabled ? 'disabled' : null"
(click)="onSubmit()">{{submitButtonLabel | translate}}</button>
</div>
</div>
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.less b/ambari-logsearch/ambari-logsearch-web/src/app/components/modal/modal.component.less
similarity index 90%
copy from ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.less
copy to ambari-logsearch/ambari-logsearch-web/src/app/components/modal/modal.component.less
index fff57df..9d1d642 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.less
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/modal/modal.component.less
@@ -15,10 +15,8 @@
* limitations under the License.
*/
-:host {
- display: block;
- margin-left: auto;
- menu-button {
- margin: 0 1em;
- }
+@import '../variables';
+
+.modal-xl {
+ width: @large-modal-width;
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/modal/modal.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/modal/modal.component.ts
index a5df53f..a724086 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/modal/modal.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/modal/modal.component.ts
@@ -21,7 +21,8 @@ import * as $ from 'jquery';
@Component({
selector: 'modal',
- templateUrl: './modal.component.html'
+ templateUrl: './modal.component.html',
+ styleUrls: ['./modal.component.less']
})
export class ModalComponent implements OnInit, AfterViewInit {
@@ -81,6 +82,12 @@ export class ModalComponent implements OnInit, AfterViewInit {
@Input()
isLargeModal: boolean = false;
+ @Input()
+ isExtraLargeModal: boolean = false;
+
+ @Input()
+ isSubmitDisabled: boolean = false;
+
@ContentChild(TemplateRef)
bodyTemplate;
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/pagination/pagination.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/pagination/pagination.component.html
index 02ed84b..ecfa75d 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/pagination/pagination.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/pagination/pagination.component.html
@@ -17,7 +17,8 @@
<form class="pagination-form" [formGroup]="filtersForm">
<filter-dropdown label="{{filterInstance.label | translate}}" formControlName="pageSize"
- [options]="filterInstance.options" [isRightAlign]="true" [isDropup]="true"></filter-dropdown>
+ [options]="filterInstance.options" [isRightAlign]="true" [isDropup]="true"
+ [showCommonLabelWithSelection]="true"></filter-dropdown>
<span>{{'pagination.numbers' | translate: numbersTranslateParams}}</span>
<pagination-controls formControlName="page" [totalCount]="totalCount" [pagesCount]="pagesCount"
(currentPageChange)="setCurrentPage($event)"></pagination-controls>
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/search-box/search-box.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/search-box/search-box.component.ts
index 9dd8e26..ade57e5 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/search-box/search-box.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/search-box/search-box.component.ts
@@ -278,7 +278,7 @@ export class SearchBoxComponent implements OnInit, OnDestroy, ControlValueAccess
updateValue = (): void => {
this.currentValue = '';
if (this.onChange) {
- this.onChange(this.parameters);
+ this.onChange(this.parameters.slice());
}
};
@@ -299,7 +299,7 @@ export class SearchBoxComponent implements OnInit, OnDestroy, ControlValueAccess
}
writeValue(parameters: SearchBoxParameterProcessed[] = []): void {
- this.parameters = parameters;
+ this.parameters = parameters.slice();
this.updateValueSubject.next();
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.html
index 3cd829e..3110d9a 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.html
@@ -23,12 +23,13 @@
{{'logs.brokenListLayoutMessage' | translate}}
</div>
<form *ngIf="logs && logs.length" [formGroup]="filtersForm">
- <filter-dropdown label="{{filters.serviceLogsSorting.label | translate}}"
- formControlName="serviceLogsSorting" [options]="filters.serviceLogsSorting.options" [isRightAlign]="true">
- </filter-dropdown>
+ <filter-dropdown formControlName="serviceLogsSorting" label="{{filters.serviceLogsSorting.label | translate}}"
+ [showCommonLabelWithSelection]="true" [options]="filters.serviceLogsSorting.options"
+ [isRightAlign]="true"></filter-dropdown>
</form>
<dropdown-button label="{{'logs.columns' | translate}}" [options]="columns" [isRightAlign]="true"
- [isMultipleChoice]="true" action="updateSelectedColumns" [additionalArgs]="logsTypeMapObject.fieldsModel">
+ [isMultipleChoice]="true" (selectItem)="updateSelectedColumns($event)"
+ [listItemArguments]="logsTypeMapObject.fieldsModel">
</dropdown-button>
<div class="layout-btn-group">
<a *ngIf="layout==='FLEX'" class="btn" (click)="toggleShowLabels()" tooltip="{{'logs.toggleLabels' | translate}}">
@@ -67,7 +68,7 @@
<tr class="log-item-row">
<td class="log-action">
<dropdown-button iconClass="fa fa-ellipsis-v action" [hideCaret]="true" [options]="logActions"
- [additionalArgs]="[log]" [showSelectedValue]="false"></dropdown-button>
+ [listItemArguments]="[log]" [showSelectedValue]="false"></dropdown-button>
</td>
<td *ngIf="isColumnDisplayed('logtime')" class="log-time">
<time>
@@ -115,7 +116,7 @@
<div class="log-header">
<div class="log-action">
<dropdown-button iconClass="fa fa-ellipsis-v action" [hideCaret]="true" [options]="logActions"
- [additionalArgs]="[log]" [showSelectedValue]="false"></dropdown-button>
+ [listItemArguments]="[log]" [showSelectedValue]="false"></dropdown-button>
</div>
<div *ngIf="isColumnDisplayed('level')" [ngClass]="'log-level ' + log.level.toLowerCase()">
<log-level [logEntry]="log"></log-level>
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.less b/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.less
index 27fafe5..61a479e 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.less
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.less
@@ -192,7 +192,7 @@
.log-list {
color: @base-font-color;
border-bottom: 1px solid @log-list-border-color;
- font-size: @log-list-font-size;
+ font-size: @table-font-size;
.log-date-row {
background: @list-header-background-color;
padding: @log-list-row-data-padding;
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.spec.ts
index e883c99..e2afbc8 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.spec.ts
@@ -43,7 +43,6 @@ import {LogsContainerService} from '@app/services/logs-container.service';
import {UtilsService} from '@app/services/utils.service';
import {HttpClientService} from '@app/services/http-client.service';
import {ComponentGeneratorService} from '@app/services/component-generator.service';
-import {ComponentActionsService} from '@app/services/component-actions.service';
import {AuthService} from '@app/services/auth.service';
import {PaginationComponent} from '@app/components/pagination/pagination.component';
import {DropdownListComponent} from '@app/components/dropdown-list/dropdown-list.component';
@@ -113,7 +112,6 @@ describe('ServiceLogsTableComponent', () => {
ComponentsService,
HostsService,
ComponentGeneratorService,
- ComponentActionsService,
AuthService
],
schemas: [CUSTOM_ELEMENTS_SCHEMA]
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.ts
index 681149d..ee19e1c 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/service-logs-table/service-logs-table.component.ts
@@ -16,10 +16,11 @@
* limitations under the License.
*/
-import {Component, AfterViewInit, AfterViewChecked, ViewChild, ElementRef, Input, ChangeDetectorRef} from '@angular/core';
+import {Component, AfterViewChecked, ViewChild, ElementRef, Input, ChangeDetectorRef} from '@angular/core';
import {ListItem} from '@app/classes/list-item';
import {LogsTableComponent} from '@app/classes/components/logs-table/logs-table-component';
+import {ServiceLog} from '@app/classes/models/service-log';
import {LogsContainerService} from '@app/services/logs-container.service';
import {UtilsService} from '@app/services/utils.service';
@@ -100,53 +101,71 @@ export class ServiceLogsTableComponent extends LogsTableComponent implements Aft
readonly timeFormat: string = 'h:mm:ss A';
+ private copyLog = (log: ServiceLog): void => {
+ if (document.queryCommandSupported('copy')) {
+ const text = log.log_message,
+ node = document.createElement('textarea');
+ node.value = text;
+ Object.assign(node.style, {
+ position: 'fixed',
+ top: '0',
+ left: '0',
+ width: '1px',
+ height: '1px',
+ border: 'none',
+ outline: 'none',
+ boxShadow: 'none',
+ backgroundColor: 'transparent',
+ padding: '0'
+ });
+ document.body.appendChild(node);
+ node.select();
+ if (document.queryCommandEnabled('copy')) {
+ document.execCommand('copy');
+ } else {
+ // TODO open failed alert
+ }
+ // TODO success alert
+ document.body.removeChild(node);
+ } else {
+ // TODO failed alert
+ }
+ };
+
+ private openLog = (log: ServiceLog): void => {
+ this.logsContainer.openServiceLog(log);
+ };
+
+ private openContext = (log: ServiceLog): void => {
+ this.logsContainer.loadLogContext(log.id, log.host, log.type);
+ };
+
readonly logActions = [
{
label: 'logs.copy',
iconClass: 'fa fa-files-o',
- action: 'copyLog'
+ onSelect: this.copyLog
},
{
label: 'logs.open',
iconClass: 'fa fa-external-link',
- action: 'openLog'
+ onSelect: this.openLog
},
{
label: 'logs.context',
iconClass: 'fa fa-crosshairs',
- action: 'openContext'
+ onSelect: this.openContext
}
];
readonly customStyledColumns: string[] = ['level', 'type', 'logtime', 'log_message', 'path'];
- get contextMenuItems(): ListItem[] {
- return this.logsContainer.queryContextMenuItems;
- }
-
private readonly messageFilterParameterName: string = 'log_message';
- /**
- * The goal is to show or hide the context menu on right click.
- * @type {boolean}
- */
- private isContextMenuDisplayed: boolean = false;
-
- /**
- * 'left' CSS property value for context menu dropdown
- * @type {number}
- */
- private contextMenuLeft: number = 0;
-
- /**
- * 'top' CSS property value for context menu dropdown
- * @type {number}
- */
- private contextMenuTop:number = 0;
+ private readonly logsType: string = 'serviceLogs';
private selectedText: string = '';
-
/**
* This is a private flag to store the table layout check result. It is used to show user notifications about
* non-visible information.
@@ -154,6 +173,10 @@ export class ServiceLogsTableComponent extends LogsTableComponent implements Aft
*/
private tooManyColumnsSelected: boolean = false;
+ get contextMenuItems(): ListItem[] {
+ return this.logsContainer.queryContextMenuItems;
+ }
+
get timeZone(): string {
return this.logsContainer.timeZone;
}
@@ -166,6 +189,22 @@ export class ServiceLogsTableComponent extends LogsTableComponent implements Aft
return this.logsContainer.logsTypeMap.serviceLogs;
}
+ get isContextMenuDisplayed(): boolean {
+ return Boolean(this.selectedText);
+ };
+
+ /**
+ * 'left' CSS property value for context menu dropdown
+ * @type {number}
+ */
+ contextMenuLeft: number = 0;
+
+ /**
+ * 'top' CSS property value for context menu dropdown
+ * @type {number}
+ */
+ contextMenuTop: number = 0;
+
isDifferentDates(dateA, dateB): boolean {
return this.utils.isDifferentDates(dateA, dateB, this.timeZone);
}
@@ -173,7 +212,6 @@ export class ServiceLogsTableComponent extends LogsTableComponent implements Aft
openMessageContextMenu(event: MouseEvent): void {
const selectedText = getSelection().toString();
if (selectedText) {
- this.isContextMenuDisplayed = true;
this.contextMenuLeft = event.clientX;
this.contextMenuTop = event.clientY;
this.selectedText = selectedText;
@@ -192,8 +230,7 @@ export class ServiceLogsTableComponent extends LogsTableComponent implements Aft
/**
* Handle the event when the contextual menu component hide itself.
*/
- private onContextMenuDismiss = (): void => {
- this.isContextMenuDisplayed = false;
+ onContextMenuDismiss(): void {
this.selectedText = '';
};
@@ -264,4 +301,8 @@ export class ServiceLogsTableComponent extends LogsTableComponent implements Aft
this.showLabels = !this.showLabels;
}
+ updateSelectedColumns(columns: string[]): void {
+ this.logsContainer.updateSelectedColumns(columns, this.logsType);
+ }
+
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/time-range-picker/time-range-picker.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/time-range-picker/time-range-picker.component.spec.ts
index e3034b0..b41760f 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/time-range-picker/time-range-picker.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/time-range-picker/time-range-picker.component.spec.ts
@@ -37,6 +37,7 @@ import {ServiceLogsTruncatedService, serviceLogsTruncated} from '@app/services/s
import {TabsService, tabs} from '@app/services/storage/tabs.service';
import {HttpClientService} from '@app/services/http-client.service';
import {LogsContainerService} from '@app/services/logs-container.service';
+import {UtilsService} from '@app/services/utils.service';
import {TimeRangePickerComponent} from './time-range-picker.component';
@@ -79,6 +80,7 @@ describe('TimeRangePickerComponent', () => {
useValue: httpClient
},
LogsContainerService,
+ UtilsService,
AppSettingsService,
AppStateService,
ClustersService,
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/time-range-picker/time-range-picker.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/time-range-picker/time-range-picker.component.ts
index 74a2b2d..6c71a26 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/time-range-picker/time-range-picker.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/time-range-picker/time-range-picker.component.ts
@@ -79,7 +79,7 @@ export class TimeRangePickerComponent implements ControlValueAccessor {
setCustomTimeRange(): void {
this.selection = {
- label: 'filter.timeRange.custom',
+ label: this.logsContainer.customTimeRangeKey,
value: {
type: 'CUSTOM',
start: this.startTime,
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/timezone-picker/timezone-picker.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/timezone-picker/timezone-picker.component.spec.ts
index 1772ec0..b522562 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/timezone-picker/timezone-picker.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/timezone-picker/timezone-picker.component.spec.ts
@@ -34,9 +34,10 @@ import {
} from '@app/services/storage/service-logs-histogram-data.service';
import {ServiceLogsTruncatedService, serviceLogsTruncated} from '@app/services/storage/service-logs-truncated.service';
import {TabsService, tabs} from '@app/services/storage/tabs.service';
-import {ComponentActionsService} from '@app/services/component-actions.service';
import {HttpClientService} from '@app/services/http-client.service';
import {LogsContainerService} from '@app/services/logs-container.service';
+import {UserSettingsService} from '@app/services/user-settings.service';
+import {UtilsService} from '@app/services/utils.service';
import {AuthService} from '@app/services/auth.service';
import {TimeZoneAbbrPipe} from '@app/pipes/timezone-abbr.pipe';
import {ModalComponent} from '@app/components/modal/modal.component';
@@ -94,13 +95,14 @@ describe('TimeZonePickerComponent', () => {
ServiceLogsHistogramDataService,
ServiceLogsTruncatedService,
TabsService,
- ComponentActionsService,
{
provide: HttpClientService,
useValue: httpClient
},
LogsContainerService,
- AuthService
+ AuthService,
+ UserSettingsService,
+ UtilsService
],
})
.compileComponents();
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/timezone-picker/timezone-picker.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/timezone-picker/timezone-picker.component.ts
index 32f6474..a8a3480 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/timezone-picker/timezone-picker.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/timezone-picker/timezone-picker.component.ts
@@ -16,21 +16,24 @@
* limitations under the License.
*/
-import {Component} from '@angular/core';
+import {Component, OnInit} from '@angular/core';
import * as $ from 'jquery';
import '@vendor/js/WorldMapGenerator.min';
import {AppSettingsService} from '@app/services/storage/app-settings.service';
-import {ComponentActionsService} from '@app/services/component-actions.service';
+import {UserSettingsService} from '@app/services/user-settings.service';
@Component({
selector: 'timezone-picker',
templateUrl: './timezone-picker.component.html',
styleUrls: ['./timezone-picker.component.less']
})
-export class TimeZonePickerComponent {
+export class TimeZonePickerComponent implements OnInit {
- constructor(private appSettings: AppSettingsService, private actions: ComponentActionsService) {
- appSettings.getParameter('timeZone').subscribe(value => this.timeZone = value);
+ constructor(private appSettings: AppSettingsService, private settingsService: UserSettingsService) {
+ }
+
+ ngOnInit() {
+ this.appSettings.getParameter('timeZone').subscribe((value: string) => this.timeZone = value);
}
readonly mapElementId = 'timezone-map';
@@ -70,7 +73,7 @@ export class TimeZonePickerComponent {
setTimeZone(): void {
const timeZone = this.timeZoneSelect.val();
- this.actions.setTimeZone(timeZone);
+ this.settingsService.setTimeZone(timeZone);
this.setTimeZonePickerDisplay(false);
}
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/top-menu/top-menu.component.html b/ambari-logsearch/ambari-logsearch-web/src/app/components/top-menu/top-menu.component.html
index 910e55f..71637bb 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/top-menu/top-menu.component.html
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/top-menu/top-menu.component.html
@@ -19,10 +19,10 @@
<form [formGroup]="filtersForm" class="filters">
<filter-dropdown *ngIf="isClustersFilterDisplayed" formControlName="clusters" [options]="filters.clusters.options"
[isMultipleChoice]="true" label="{{filters.clusters.label | translate}}" [isRightAlign]="true"
- buttonClass="inherited-color"></filter-dropdown>
+ buttonClass="btn-link inherited-color"></filter-dropdown>
</form>
- <menu-button *ngFor="let item of items" label="{{item.label | translate}}" [action]="item.action"
- [iconClass]="item.iconClass" [labelClass]="item.labelClass" [subItems]="item.subItems"
- [hideCaret]="item.hideCaret" [badge]="item.badge" [isRightAlign]="item.isRightAlign">
+ <menu-button *ngFor="let item of items" label="{{item.label | translate}}" [iconClass]="item.iconClass"
+ [labelClass]="item.labelClass" [subItems]="item.subItems" [hideCaret]="item.hideCaret"
+ [badge]="item.badge" [isRightAlign]="item.isRightAlign">
</menu-button>
</div>
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/top-menu/top-menu.component.spec.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/top-menu/top-menu.component.spec.ts
index ce4fa1c..caeb197 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/top-menu/top-menu.component.spec.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/top-menu/top-menu.component.spec.ts
@@ -38,6 +38,8 @@ import {ComponentsService, components} from '@app/services/storage/components.se
import {HostsService, hosts} from '@app/services/storage/hosts.service';
import {LogsContainerService} from '@app/services/logs-container.service';
import {HttpClientService} from '@app/services/http-client.service';
+import {AuthService} from '@app/services/auth.service';
+import {UtilsService} from '@app/services/utils.service';
import {TopMenuComponent} from './top-menu.component';
@@ -81,6 +83,8 @@ describe('TopMenuComponent', () => {
provide: HttpClientService,
useValue: httpClient
},
+ AuthService,
+ UtilsService,
AuditLogsService,
ServiceLogsService,
AuditLogsFieldsService,
diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/components/top-menu/top-menu.component.ts b/ambari-logsearch/ambari-logsearch-web/src/app/components/top-menu/top-menu.component.ts
index 8d739ec..1d003ee 100644
--- a/ambari-logsearch/ambari-logsearch-web/src/app/components/top-menu/top-menu.component.ts
+++ b/ambari-logsearch/ambari-logsearch-web/src/app/components/top-menu/top-menu.component.ts
@@ -21,6 +21,7 @@ import {FormGroup} from '@angular/forms';
import {FilterCondition, TimeUnitListItem} from '@app/classes/filtering';
import {ListItem} from '@app/classes/list-item';
import {HomogeneousObject} from '@app/classes/object';
+import {AuthService} from '@app/services/auth.service';
import {LogsContainerService} from '@app/services/logs-container.service';
@Component({
@@ -30,7 +31,7 @@ import {LogsContainerService} from '@app/services/logs-container.service';
})
export class TopMenuComponent {
- constructor(private logsContainer: LogsContainerService) {
+ constructor(private authService: AuthService, private logsContainer: LogsContainerService) {
... 146916 lines suppressed ...
--
To stop receiving notification emails like this one, please contact
dsen@apache.org.