You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2017/11/11 11:01:55 UTC
[12/50] [abbrv] hadoop git commit: Merge branch
'yarn-native-services' into trunk
Merge branch 'yarn-native-services' into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbc632d9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbc632d9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbc632d9
Branch: refs/heads/YARN-5881
Commit: cbc632d9abf08c56a7fc02be51b2718af30bad28
Parents: dcd99c4 a55d073
Author: Jian He <ji...@apache.org>
Authored: Mon Nov 6 14:02:19 2017 -0800
Committer: Jian He <ji...@apache.org>
Committed: Mon Nov 6 14:02:19 2017 -0800
----------------------------------------------------------------------
LICENSE.txt | 1 +
NOTICE.txt | 10 +
.../resources/assemblies/hadoop-yarn-dist.xml | 25 +
hadoop-project/pom.xml | 44 +-
hadoop-project/src/site/site.xml | 10 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 18 +-
.../hadoop-yarn/conf/yarn-env.sh | 18 +
.../dev-support/findbugs-exclude.xml | 16 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 2 +
.../dev-support/findbugs-exclude.xml | 20 +
.../hadoop-yarn-services-api/pom.xml | 130 ++
.../hadoop/yarn/service/webapp/ApiServer.java | 298 +++
.../yarn/service/webapp/ApiServerWebApp.java | 161 ++
.../definition/YARN-Services-Examples.md | 245 +++
...RN-Simplified-V1-API-Layer-For-Services.yaml | 471 +++++
.../src/main/resources/log4j-server.properties | 76 +
.../src/main/resources/webapps/api-server/app | 16 +
.../src/main/webapp/WEB-INF/web.xml | 36 +
.../hadoop/yarn/service/ServiceClientTest.java | 107 ++
.../hadoop/yarn/service/TestApiServer.java | 366 ++++
.../dev-support/findbugs-exclude.xml | 48 +
.../conf/yarnservice-log4j.properties | 62 +
.../examples/httpd-no-dns/httpd-no-dns.json | 62 +
.../httpd-no-dns/httpd-proxy-no-dns.conf | 24 +
.../examples/httpd/httpd-proxy.conf | 24 +
.../examples/httpd/httpd.json | 55 +
.../examples/sleeper/sleeper.json | 15 +
.../hadoop-yarn-services-core/pom.xml | 255 +++
.../hadoop/yarn/service/ClientAMProtocol.java | 40 +
.../hadoop/yarn/service/ClientAMService.java | 132 ++
.../yarn/service/ContainerFailureTracker.java | 89 +
.../hadoop/yarn/service/ServiceContext.java | 41 +
.../hadoop/yarn/service/ServiceMaster.java | 169 ++
.../hadoop/yarn/service/ServiceMetrics.java | 94 +
.../hadoop/yarn/service/ServiceScheduler.java | 691 +++++++
.../yarn/service/api/ServiceApiConstants.java | 74 +
.../yarn/service/api/records/Artifact.java | 168 ++
.../yarn/service/api/records/BaseResource.java | 52 +
.../yarn/service/api/records/Component.java | 430 +++++
.../service/api/records/ComponentState.java | 30 +
.../yarn/service/api/records/ConfigFile.java | 233 +++
.../yarn/service/api/records/ConfigFormat.java | 67 +
.../yarn/service/api/records/Configuration.java | 225 +++
.../yarn/service/api/records/Container.java | 298 +++
.../service/api/records/ContainerState.java | 30 +
.../hadoop/yarn/service/api/records/Error.java | 129 ++
.../service/api/records/PlacementPolicy.java | 102 +
.../service/api/records/ReadinessCheck.java | 183 ++
.../yarn/service/api/records/Resource.java | 161 ++
.../yarn/service/api/records/Service.java | 390 ++++
.../yarn/service/api/records/ServiceState.java | 33 +
.../yarn/service/api/records/ServiceStatus.java | 148 ++
.../yarn/service/client/ClientAMProxy.java | 57 +
.../yarn/service/client/ServiceClient.java | 960 ++++++++++
.../yarn/service/component/Component.java | 584 ++++++
.../yarn/service/component/ComponentEvent.java | 83 +
.../service/component/ComponentEventType.java | 27 +
.../yarn/service/component/ComponentState.java | 25 +
.../component/instance/ComponentInstance.java | 549 ++++++
.../instance/ComponentInstanceEvent.java | 58 +
.../instance/ComponentInstanceEventType.java | 26 +
.../component/instance/ComponentInstanceId.java | 91 +
.../instance/ComponentInstanceState.java | 26 +
.../yarn/service/conf/RestApiConstants.java | 39 +
.../yarn/service/conf/SliderExitCodes.java | 88 +
.../yarn/service/conf/YarnServiceConf.java | 113 ++
.../yarn/service/conf/YarnServiceConstants.java | 92 +
.../containerlaunch/AbstractLauncher.java | 271 +++
.../containerlaunch/ClasspathConstructor.java | 172 ++
.../containerlaunch/CommandLineBuilder.java | 86 +
.../containerlaunch/ContainerLaunchService.java | 101 +
.../containerlaunch/CredentialUtils.java | 319 ++++
.../containerlaunch/JavaCommandLineBuilder.java | 181 ++
.../exceptions/BadClusterStateException.java | 36 +
.../BadCommandArgumentsException.java | 30 +
.../service/exceptions/BadConfigException.java | 39 +
.../yarn/service/exceptions/ErrorStrings.java | 42 +
.../service/exceptions/ExitCodeProvider.java | 32 +
.../service/exceptions/LauncherExitCodes.java | 196 ++
.../exceptions/RestApiErrorMessages.java | 92 +
.../exceptions/ServiceLaunchException.java | 73 +
.../service/exceptions/SliderException.java | 66 +
.../yarn/service/exceptions/UsageException.java | 34 +
.../pb/client/ClientAMProtocolPBClientImpl.java | 91 +
.../impl/pb/service/ClientAMProtocolPB.java | 29 +
.../service/ClientAMProtocolPBServiceImpl.java | 70 +
.../yarn/service/monitor/ServiceMonitor.java | 147 ++
.../yarn/service/monitor/probe/HttpProbe.java | 110 ++
.../service/monitor/probe/LogEntryBuilder.java | 76 +
.../yarn/service/monitor/probe/MonitorKeys.java | 66 +
.../service/monitor/probe/MonitorUtils.java | 84 +
.../yarn/service/monitor/probe/PortProbe.java | 98 +
.../yarn/service/monitor/probe/Probe.java | 100 +
.../yarn/service/monitor/probe/ProbeStatus.java | 160 ++
.../provider/AbstractClientProvider.java | 129 ++
.../provider/AbstractProviderService.java | 113 ++
.../yarn/service/provider/ProviderFactory.java | 76 +
.../yarn/service/provider/ProviderService.java | 39 +
.../yarn/service/provider/ProviderUtils.java | 408 ++++
.../defaultImpl/DefaultClientProvider.java | 46 +
.../defaultImpl/DefaultProviderFactory.java | 51 +
.../defaultImpl/DefaultProviderService.java | 36 +
.../provider/docker/DockerClientProvider.java | 53 +
.../service/provider/docker/DockerKeys.java | 30 +
.../provider/docker/DockerProviderFactory.java | 52 +
.../provider/docker/DockerProviderService.java | 57 +
.../provider/tarball/TarballClientProvider.java | 65 +
.../tarball/TarballProviderFactory.java | 52 +
.../tarball/TarballProviderService.java | 48 +
.../registry/CustomRegistryConstants.java | 57 +
.../registry/YarnRegistryViewForProviders.java | 249 +++
.../timelineservice/ServiceMetricsSink.java | 102 +
.../ServiceTimelineEntityType.java | 39 +
.../timelineservice/ServiceTimelineEvent.java | 36 +
.../ServiceTimelineMetricsConstants.java | 92 +
.../ServiceTimelinePublisher.java | 381 ++++
.../service/timelineservice/package-info.java | 27 +
.../utils/ApplicationReportSerDeser.java | 56 +
.../service/utils/ClientRegistryBinder.java | 201 ++
.../hadoop/yarn/service/utils/Comparators.java | 62 +
.../hadoop/yarn/service/utils/ConfigHelper.java | 157 ++
.../hadoop/yarn/service/utils/ConfigUtils.java | 97 +
.../yarn/service/utils/CoreFileSystem.java | 521 ++++++
.../hadoop/yarn/service/utils/Duration.java | 109 ++
.../hadoop/yarn/service/utils/JsonSerDeser.java | 249 +++
.../yarn/service/utils/PatternValidator.java | 58 +
.../hadoop/yarn/service/utils/PortScanner.java | 113 ++
.../service/utils/PublishedConfiguration.java | 196 ++
.../utils/PublishedConfigurationOutputter.java | 212 +++
.../utils/SerializedApplicationReport.java | 98 +
.../yarn/service/utils/ServiceApiUtil.java | 401 ++++
.../service/utils/ServiceRegistryUtils.java | 71 +
.../hadoop/yarn/service/utils/ServiceUtils.java | 545 ++++++
.../yarn/service/utils/SliderFileSystem.java | 51 +
.../yarn/service/utils/ZookeeperUtils.java | 146 ++
.../src/main/proto/ClientAMProtocol.proto | 56 +
.../hadoop/yarn/service/MockServiceAM.java | 221 +++
.../hadoop/yarn/service/ServiceTestUtils.java | 262 +++
.../hadoop/yarn/service/TestServiceApiUtil.java | 480 +++++
.../yarn/service/TestYarnNativeServices.java | 403 ++++
.../client/TestBuildExternalComponents.java | 119 ++
.../yarn/service/client/TestServiceCLI.java | 127 ++
.../yarn/service/conf/ExampleAppJson.java | 65 +
.../yarn/service/conf/TestAppJsonResolve.java | 200 ++
.../service/conf/TestLoadExampleAppJson.java | 71 +
.../service/conf/TestValidateServiceNames.java | 125 ++
.../service/monitor/TestServiceMonitor.java | 104 ++
.../providers/TestAbstractClientProvider.java | 118 ++
.../service/providers/TestProviderFactory.java | 76 +
.../TestServiceTimelinePublisher.java | 291 +++
.../src/test/resources/example-app.json | 15 +
.../service/conf/examples/app-override.json | 76 +
.../hadoop/yarn/service/conf/examples/app.json | 50 +
.../yarn/service/conf/examples/default.json | 16 +
.../yarn/service/conf/examples/external0.json | 15 +
.../yarn/service/conf/examples/external1.json | 30 +
.../yarn/service/conf/examples/external2.json | 22 +
.../src/test/resources/yarn-site.xml | 19 +
.../hadoop-yarn-services/pom.xml | 38 +
.../hadoop-yarn-applications/pom.xml | 2 +
.../hadoop/yarn/client/api/AppAdminClient.java | 222 +++
.../hadoop/yarn/client/cli/ApplicationCLI.java | 318 +++-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 158 +-
.../yarn/security/ContainerTokenIdentifier.java | 20 +-
.../java/org/apache/hadoop/yarn/util/Apps.java | 36 +
.../src/main/proto/yarn_security_token.proto | 1 +
.../src/main/resources/yarn-default.xml | 8 +
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 78 +-
.../registry/client/api/DNSOperations.java | 60 +
.../client/api/DNSOperationsFactory.java | 78 +
.../registry/client/api/RegistryConstants.java | 136 +-
.../client/api/RegistryOperationsFactory.java | 8 +
.../client/binding/RegistryTypeUtils.java | 3 +-
.../registry/client/impl/zk/CuratorService.java | 271 ++-
.../registry/client/impl/zk/ListenerHandle.java | 25 +
.../registry/client/impl/zk/PathListener.java | 30 +
.../impl/zk/RegistryOperationsService.java | 4 +-
.../types/yarn/YarnRegistryAttributes.java | 16 +-
.../dns/ApplicationServiceRecordProcessor.java | 366 ++++
.../server/dns/BaseServiceRecordProcessor.java | 468 +++++
.../dns/ContainerServiceRecordProcessor.java | 283 +++
.../dns/PrivilegedRegistryDNSStarter.java | 80 +
.../server/dns/RecordCreatorFactory.java | 275 +++
.../hadoop/registry/server/dns/RegistryDNS.java | 1756 ++++++++++++++++++
.../registry/server/dns/RegistryDNSServer.java | 267 +++
.../registry/server/dns/ReverseZoneUtils.java | 171 ++
.../registry/server/dns/SecureableZone.java | 151 ++
.../server/dns/ServiceRecordProcessor.java | 53 +
.../registry/server/dns/ZoneSelector.java | 33 +
.../registry/server/dns/package-info.java | 26 +
.../registry/server/dns/TestRegistryDNS.java | 627 +++++++
.../server/dns/TestReverseZoneUtils.java | 89 +
.../server/dns/TestSecureRegistryDNS.java | 44 +
.../test/resources/0.17.172.in-addr.arpa.zone | 36 +
.../src/test/resources/test.private | 32 +
.../api/protocolrecords/NMContainerStatus.java | 18 +-
.../impl/pb/NMContainerStatusPBImpl.java | 15 +-
.../OpportunisticContainerAllocator.java | 2 +-
.../yarn_server_common_service_protos.proto | 1 +
.../containermanager/ContainerManagerImpl.java | 5 +-
.../container/ContainerImpl.java | 8 +-
.../runtime/DockerLinuxContainerRuntime.java | 8 +
.../blacklist/SimpleBlacklistManager.java | 8 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 35 +-
.../scheduler/AbstractYarnScheduler.java | 1 +
.../scheduler/SchedulerApplicationAttempt.java | 3 +-
.../security/RMContainerTokenSecretManager.java | 36 +-
.../server/resourcemanager/webapp/RMWebApp.java | 19 +
.../server/resourcemanager/TestRMRestart.java | 2 +-
.../TestResourceTrackerService.java | 6 +-
.../capacity/TestContainerAllocation.java | 5 +-
.../src/site/markdown/YarnCommands.md | 33 +-
.../src/site/markdown/registry/index.md | 1 +
.../markdown/registry/registry-configuration.md | 4 +-
.../src/site/markdown/yarn-service/Concepts.md | 46 +
.../markdown/yarn-service/Configurations.md | 165 ++
.../src/site/markdown/yarn-service/Examples.md | 161 ++
.../src/site/markdown/yarn-service/Overview.md | 61 +
.../site/markdown/yarn-service/QuickStart.md | 217 +++
.../site/markdown/yarn-service/RegistryDNS.md | 200 ++
.../markdown/yarn-service/ServiceDiscovery.md | 136 ++
.../markdown/yarn-service/YarnServiceAPI.md | 598 ++++++
.../src/site/resources/images/dns_overview.png | Bin 0 -> 41908 bytes
.../resources/images/dns_record_creation.jpeg | Bin 0 -> 51911 bytes
.../resources/images/dns_record_removal.jpeg | Bin 0 -> 58041 bytes
.../main/webapp/app/adapters/restabstract.js | 50 +
.../app/adapters/yarn-component-instance.js | 32 +
.../app/adapters/yarn-service-component.js | 32 +
.../webapp/app/adapters/yarn-service-info.js | 32 +
.../main/webapp/app/adapters/yarn-servicedef.js | 44 +
.../webapp/app/components/breadcrumb-bar.js | 1 +
.../webapp/app/components/confirm-dialog.js | 31 +
.../webapp/app/components/deploy-service.js | 167 ++
.../app/components/fileconfig-viewer-dialog.js | 36 +
.../main/webapp/app/components/info-tooltip.js | 44 +
.../main/webapp/app/components/metrics-table.js | 24 +
.../app/components/service-component-table.js | 56 +
.../app/components/service-config-table.js | 89 +
.../app/components/service-fileconfig-table.js | 112 ++
.../main/webapp/app/components/upload-config.js | 54 +
.../app/controllers/yarn-app/components.js | 59 +
.../webapp/app/controllers/yarn-app/configs.js | 24 +
.../webapp/app/controllers/yarn-app/info.js | 60 +
.../app/controllers/yarn-component-instance.js | 59 +
.../controllers/yarn-component-instance/info.js | 25 +
.../app/controllers/yarn-component-instances.js | 59 +
.../yarn-component-instances/configs.js | 25 +
.../yarn-component-instances/info.js | 62 +
.../app/controllers/yarn-deploy-service.js | 69 +
.../webapp/app/helpers/check-availability.js | 28 +
.../app/models/yarn-component-instance.js | 51 +
.../webapp/app/models/yarn-service-component.js | 46 +
.../main/webapp/app/models/yarn-service-info.js | 57 +
.../main/webapp/app/models/yarn-servicedef.js | 278 +++
.../src/main/webapp/app/router.js | 10 +
.../webapp/app/routes/yarn-app/components.js | 49 +
.../main/webapp/app/routes/yarn-app/configs.js | 52 +
.../src/main/webapp/app/routes/yarn-app/info.js | 12 +-
.../app/routes/yarn-component-instance.js | 29 +
.../app/routes/yarn-component-instance/info.js | 45 +
.../app/routes/yarn-component-instances.js | 29 +
.../routes/yarn-component-instances/configs.js | 44 +
.../app/routes/yarn-component-instances/info.js | 53 +
.../webapp/app/routes/yarn-deploy-service.js | 27 +
.../src/main/webapp/app/routes/yarn-services.js | 2 +-
.../app/serializers/yarn-component-instance.js | 72 +
.../app/serializers/yarn-service-component.js | 77 +
.../webapp/app/serializers/yarn-service-info.js | 87 +
.../src/main/webapp/app/services/hosts.js | 4 +
.../src/main/webapp/app/styles/app.css | 176 ++
.../main/webapp/app/templates/application.hbs | 2 +
.../app/templates/components/breadcrumb-bar.hbs | 4 +-
.../app/templates/components/confirm-dialog.hbs | 37 +
.../app/templates/components/deploy-service.hbs | 157 ++
.../components/fileconfig-viewer-dialog.hbs | 53 +
.../app/templates/components/info-tooltip.hbs | 20 +
.../app/templates/components/metrics-table.hbs | 55 +
.../components/service-component-table.hbs | 113 ++
.../components/service-config-table.hbs | 130 ++
.../components/service-fileconfig-table.hbs | 152 ++
.../app/templates/components/upload-config.hbs | 44 +
.../src/main/webapp/app/templates/yarn-app.hbs | 8 +
.../app/templates/yarn-app/components.hbs | 23 +
.../webapp/app/templates/yarn-app/configs.hbs | 57 +
.../main/webapp/app/templates/yarn-app/info.hbs | 111 +-
.../app/templates/yarn-component-instance.hbs | 43 +
.../templates/yarn-component-instance/info.hbs | 77 +
.../app/templates/yarn-component-instances.hbs | 46 +
.../yarn-component-instances/configs.hbs | 53 +
.../templates/yarn-component-instances/info.hbs | 28 +
.../yarn-component-instances/loading.hbs | 23 +
.../app/templates/yarn-deploy-service.hbs | 33 +
.../main/webapp/app/templates/yarn-services.hbs | 4 +
.../src/main/webapp/app/utils/info-seeder.js | 26 +
.../src/main/webapp/config/default-config.js | 6 +-
.../components/confirm-dialog-test.js | 43 +
.../components/deploy-service-test.js | 43 +
.../components/fileconfig-viewer-dialog-test.js | 43 +
.../integration/components/info-tooltip-test.js | 43 +
.../components/metrics-table-test.js | 43 +
.../components/service-component-table-test.js | 43 +
.../components/service-config-table-test.js | 43 +
.../components/service-fileconfig-table-test.js | 43 +
.../components/upload-config-test.js | 43 +
.../adapters/yarn-component-instance-test.js | 30 +
.../adapters/yarn-service-component-test.js | 30 +
.../unit/adapters/yarn-service-info-test.js | 30 +
.../tests/unit/adapters/yarn-servicedef-test.js | 30 +
.../controllers/yarn-app/components-test.js | 30 +
.../unit/controllers/yarn-app/configs-test.js | 30 +
.../controllers/yarn-component-instance-test.js | 30 +
.../yarn-component-instance/info-test.js | 30 +
.../yarn-component-instances-test.js | 30 +
.../yarn-component-instances/configs-test.js | 30 +
.../yarn-component-instances/info-test.js | 30 +
.../controllers/yarn-deploy-service-test.js | 30 +
.../unit/helpers/check-availability-test.js | 28 +
.../unit/models/yarn-component-instance-test.js | 30 +
.../unit/models/yarn-service-component-test.js | 30 +
.../tests/unit/models/yarn-service-info-test.js | 30 +
.../tests/unit/models/yarn-servicedef-test.js | 30 +
.../unit/routes/yarn-app/components-test.js | 29 +
.../tests/unit/routes/yarn-app/configs-test.js | 29 +
.../unit/routes/yarn-component-instance-test.js | 29 +
.../routes/yarn-component-instance/info-test.js | 29 +
.../routes/yarn-component-instances-test.js | 29 +
.../yarn-component-instances/configs-test.js | 29 +
.../yarn-component-instances/info-test.js | 29 +
.../unit/routes/yarn-deploy-service-test.js | 29 +
.../serializers/yarn-component-instance-test.js | 33 +
.../serializers/yarn-service-component-test.js | 33 +
.../unit/serializers/yarn-service-info-test.js | 33 +
hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +-
hadoop-yarn-project/pom.xml | 4 +
334 files changed, 33539 insertions(+), 320 deletions(-)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org