You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dlab.apache.org by of...@apache.org on 2020/09/28 13:46:42 UTC
[incubator-dlab] branch new-datalab-name updated: [DLAB-2066]
Replaced old name with new one on backend side
This is an automated email from the ASF dual-hosted git repository.
ofuks pushed a commit to branch new-datalab-name
in repository https://gitbox.apache.org/repos/asf/incubator-dlab.git
The following commit(s) were added to refs/heads/new-datalab-name by this push:
new 6d85c49 [DLAB-2066] Replaced old name with new one on backend side
6d85c49 is described below
commit 6d85c496dff9a510402f5e4869fe4f87e09bff54
Author: Oleh Fuks <ol...@gmail.com>
AuthorDate: Mon Sep 28 16:46:18 2020 +0300
[DLAB-2066] Replaced old name with new one on backend side
---
build.properties | 2 +-
integration-tests-cucumber/pom.xml | 2 +-
.../java/org/apache/datalab/dto/EndpointDTO.java | 38 +
.../org/apache/datalab/mongo/MongoDBHelper.java | 33 +
.../org/apache/datalab/util/JacksonMapper.java | 35 +
.../org/apache/datalab/util/PropertyHelper.java | 42 +
.../main/java/org/apache/dlab/dto/EndpointDTO.java | 38 -
.../java/org/apache/dlab/mongo/MongoDBHelper.java | 33 -
.../java/org/apache/dlab/util/JacksonMapper.java | 35 -
.../java/org/apache/dlab/util/PropertyHelper.java | 42 -
.../src/test/java/datalab/Constants.java | 24 +
.../src/test/java/datalab/RunCucumberTest.java | 29 +
.../test/java/datalab/endpoint/EndpointSteps.java | 116 ++
.../src/test/java/datalab/login/LoginSteps.java | 63 ++
.../src/test/java/dlab/Constants.java | 24 -
.../src/test/java/dlab/RunCucumberTest.java | 29 -
.../src/test/java/dlab/endpoint/EndpointSteps.java | 116 --
.../src/test/java/dlab/login/LoginSteps.java | 63 --
.../src/test/resources/config.properties | 4 +-
.../src/test/resources/datalab/endpoint.feature | 75 ++
.../src/test/resources/datalab/login.feature | 30 +
.../src/test/resources/dlab/endpoint.feature | 75 --
.../src/test/resources/dlab/login.feature | 30 -
integration-tests/pom.xml | 26 +-
.../cloud/VirtualMachineStatusChecker.java | 18 +-
.../dlab/automation/cloud/aws/AmazonHelper.java | 8 +-
.../automation/cloud/aws/AmazonInstanceState.java | 2 +-
.../dlab/automation/cloud/azure/AzureHelper.java | 6 +-
.../epam/dlab/automation/cloud/gcp/GcpHelper.java | 6 +-
.../automation/cloud/gcp/GcpInstanceState.java | 2 +-
.../com/epam/dlab/automation/docker/AckStatus.java | 2 +-
.../com/epam/dlab/automation/docker/Bridge.java | 2 +-
.../com/epam/dlab/automation/docker/Docker.java | 6 +-
.../dlab/automation/docker/DockerContainer.java | 2 +-
.../epam/dlab/automation/docker/HostConfig.java | 2 +-
.../com/epam/dlab/automation/docker/Labels.java | 2 +-
.../dlab/automation/docker/NetworkSettings.java | 2 +-
.../com/epam/dlab/automation/docker/Networks.java | 2 +-
.../epam/dlab/automation/docker/SSHConnect.java | 4 +-
.../dlab/automation/exceptions/CloudException.java | 2 +-
.../automation/exceptions/DockerException.java | 2 +-
.../automation/exceptions/JenkinsException.java | 2 +-
.../automation/exceptions/LoadFailException.java | 2 +-
.../epam/dlab/automation/helper/CloudHelper.java | 18 +-
.../epam/dlab/automation/helper/CloudProvider.java | 2 +-
.../automation/helper/ConfigPropertyValue.java | 4 +-
.../epam/dlab/automation/helper/NamingHelper.java | 2 +-
.../dlab/automation/helper/PropertiesResolver.java | 2 +-
.../epam/dlab/automation/helper/WaitForStatus.java | 8 +-
.../com/epam/dlab/automation/http/ApiPath.java | 2 +-
.../com/epam/dlab/automation/http/ContentType.java | 2 +-
.../com/epam/dlab/automation/http/HttpRequest.java | 4 +-
.../epam/dlab/automation/http/HttpStatusCode.java | 2 +-
.../jenkins/JenkinsConfigProperties.java | 2 +-
.../jenkins/JenkinsResponseElements.java | 2 +-
.../dlab/automation/jenkins/JenkinsService.java | 10 +-
.../epam/dlab/automation/jenkins/JenkinsUrls.java | 2 +-
.../dlab/automation/model/CreateNotebookDto.java | 2 +-
.../dlab/automation/model/DeployClusterDto.java | 2 +-
.../dlab/automation/model/DeployDataProcDto.java | 2 +-
.../epam/dlab/automation/model/DeployEMRDto.java | 2 +-
.../epam/dlab/automation/model/DeploySparkDto.java | 2 +-
.../dlab/automation/model/ExploratoryImageDto.java | 2 +-
.../com/epam/dlab/automation/model/ImageDto.java | 2 +-
.../epam/dlab/automation/model/JsonMapperDto.java | 2 +-
.../java/com/epam/dlab/automation/model/Lib.java | 2 +-
.../com/epam/dlab/automation/model/LoginDto.java | 2 +-
.../epam/dlab/automation/model/NotebookConfig.java | 2 +-
integration-tests/src/main/resources/log4j2.xml | 18 +-
.../epam/dlab/automation/test/TestCallable.java | 32 +-
.../automation/test/TestDataEngineService.java | 16 +-
.../epam/dlab/automation/test/TestServices.java | 26 +-
.../test/libs/LibraryNotFoundException.java | 2 +-
.../epam/dlab/automation/test/libs/LibsHelper.java | 6 +-
.../dlab/automation/test/libs/TestDescription.java | 2 +-
.../automation/test/libs/TestLibGroupStep.java | 12 +-
.../automation/test/libs/TestLibInstallStep.java | 22 +-
.../dlab/automation/test/libs/TestLibListStep.java | 20 +-
.../dlab/automation/test/libs/TestLibStep.java | 2 +-
.../test/libs/models/LibInstallRequest.java | 4 +-
.../test/libs/models/LibSearchRequest.java | 2 +-
.../test/libs/models/LibStatusResponse.java | 2 +-
.../test/libs/models/LibToSearchData.java | 2 +-
.../automation/test/libs/models/LibraryStatus.java | 2 +-
integration-tests/src/test/resources/log4j2.xml | 18 +-
pom.xml | 52 +-
services/billing-aws/billing.yml | 16 +-
services/billing-aws/pom.xml | 10 +-
.../com/epam/datalab/BillingAwsApplication.java | 37 +
.../main/java/com/epam/datalab/BillingService.java | 29 +
.../java/com/epam/datalab/BillingServiceImpl.java | 127 +++
.../main/java/com/epam/datalab/BillingTool.java | 176 +++
.../src/main/java/com/epam/datalab/Help.java | 146 +++
.../java/com/epam/datalab/conf/SecurityConfig.java | 66 ++
.../configuration/BillingToolConfiguration.java | 282 +++++
.../BillingToolConfigurationFactory.java | 110 ++
.../datalab/configuration/ConfigJsonGenerator.java | 103 ++
.../configuration/ConfigurationValidator.java | 71 ++
.../datalab/configuration/FreeMarkerConfig.java | 84 ++
.../configuration/LoggingConfigurationFactory.java | 170 +++
.../epam/datalab/controller/BillingController.java | 44 +
.../java/com/epam/datalab/core/AdapterBase.java | 173 +++
.../java/com/epam/datalab/core/BillingUtils.java | 144 +++
.../java/com/epam/datalab/core/DBAdapterBase.java | 163 +++
.../java/com/epam/datalab/core/FilterBase.java | 95 ++
.../java/com/epam/datalab/core/ModuleBase.java | 84 ++
.../java/com/epam/datalab/core/ModuleData.java | 182 ++++
.../java/com/epam/datalab/core/ModuleType.java | 46 +
.../core/aggregate/AggregateGranularity.java | 44 +
.../datalab/core/aggregate/DataAggregator.java | 172 +++
.../epam/datalab/core/aggregate/UsageDataList.java | 87 ++
.../com/epam/datalab/core/parser/ColumnInfo.java | 59 +
.../com/epam/datalab/core/parser/ColumnMeta.java | 318 ++++++
.../com/epam/datalab/core/parser/CommonFormat.java | 304 ++++++
.../datalab/core/parser/ConditionEvaluate.java | 168 +++
.../com/epam/datalab/core/parser/ParserBase.java | 395 +++++++
.../com/epam/datalab/core/parser/ParserByLine.java | 249 +++++
.../epam/datalab/core/parser/ParserStatistics.java | 194 ++++
.../com/epam/datalab/logging/AppenderBase.java | 103 ++
.../com/epam/datalab/logging/AppenderConsole.java | 44 +
.../com/epam/datalab/logging/AppenderFile.java | 208 ++++
.../com/epam/datalab/module/AdapterConsole.java | 92 ++
.../java/com/epam/datalab/module/AdapterFile.java | 164 +++
.../java/com/epam/datalab/module/ModuleName.java | 33 +
.../java/com/epam/datalab/module/ParserCsv.java | 312 ++++++
.../com/epam/datalab/module/aws/AdapterS3File.java | 388 +++++++
.../com/epam/datalab/module/aws/FilterAWS.java | 150 +++
.../com/epam/datalab/module/aws/S3FileList.java | 168 +++
.../com/epam/datalab/mongo/AdapterMongoDb.java | 229 ++++
.../epam/datalab/mongo/DatalabResourceTypeDAO.java | 115 ++
.../com/epam/datalab/mongo/MongoConstants.java | 32 +
.../com/epam/datalab/mongo/MongoDbConnection.java | 223 ++++
.../java/com/epam/dlab/BillingAwsApplication.java | 37 -
.../main/java/com/epam/dlab/BillingService.java | 29 -
.../java/com/epam/dlab/BillingServiceImpl.java | 127 ---
.../src/main/java/com/epam/dlab/BillingTool.java | 164 ---
.../src/main/java/com/epam/dlab/Help.java | 136 ---
.../java/com/epam/dlab/conf/SecurityConfig.java | 66 --
.../configuration/BillingToolConfiguration.java | 282 -----
.../BillingToolConfigurationFactory.java | 101 --
.../dlab/configuration/ConfigJsonGenerator.java | 88 --
.../dlab/configuration/ConfigurationValidator.java | 63 --
.../epam/dlab/configuration/FreeMarkerConfig.java | 81 --
.../configuration/LoggingConfigurationFactory.java | 145 ---
.../epam/dlab/controller/BillingController.java | 44 -
.../main/java/com/epam/dlab/core/AdapterBase.java | 173 ---
.../main/java/com/epam/dlab/core/BillingUtils.java | 129 ---
.../java/com/epam/dlab/core/DBAdapterBase.java | 127 ---
.../main/java/com/epam/dlab/core/FilterBase.java | 95 --
.../main/java/com/epam/dlab/core/ModuleBase.java | 73 --
.../main/java/com/epam/dlab/core/ModuleData.java | 182 ----
.../main/java/com/epam/dlab/core/ModuleType.java | 45 -
.../dlab/core/aggregate/AggregateGranularity.java | 42 -
.../epam/dlab/core/aggregate/DataAggregator.java | 172 ---
.../epam/dlab/core/aggregate/UsageDataList.java | 75 --
.../java/com/epam/dlab/core/parser/ColumnInfo.java | 50 -
.../java/com/epam/dlab/core/parser/ColumnMeta.java | 318 ------
.../com/epam/dlab/core/parser/CommonFormat.java | 304 ------
.../epam/dlab/core/parser/ConditionEvaluate.java | 152 ---
.../java/com/epam/dlab/core/parser/ParserBase.java | 311 ------
.../com/epam/dlab/core/parser/ParserByLine.java | 249 -----
.../epam/dlab/core/parser/ParserStatistics.java | 153 ---
.../java/com/epam/dlab/logging/AppenderBase.java | 93 --
.../com/epam/dlab/logging/AppenderConsole.java | 44 -
.../java/com/epam/dlab/logging/AppenderFile.java | 181 ----
.../java/com/epam/dlab/module/AdapterConsole.java | 92 --
.../java/com/epam/dlab/module/AdapterFile.java | 164 ---
.../main/java/com/epam/dlab/module/ModuleName.java | 32 -
.../main/java/com/epam/dlab/module/ParserCsv.java | 272 -----
.../com/epam/dlab/module/aws/AdapterS3File.java | 388 -------
.../java/com/epam/dlab/module/aws/FilterAWS.java | 150 ---
.../java/com/epam/dlab/module/aws/S3FileList.java | 168 ---
.../java/com/epam/dlab/mongo/AdapterMongoDb.java | 248 -----
.../com/epam/dlab/mongo/DlabResourceTypeDAO.java | 189 ----
.../java/com/epam/dlab/mongo/MongoConstants.java | 59 -
.../com/epam/dlab/mongo/MongoDbConnection.java | 223 ----
.../billing-aws/src/main/resources/application.yml | 6 +-
...b.configuration.BillingToolConfigurationFactory | 8 +
.../src/main/resources/com.epam.dlab.Help.conf.txt | 2 +-
...b.configuration.BillingToolConfigurationFactory | 8 -
.../BillingToolConfigurationTest.java | 97 ++
.../configuration/ConfigJsonGeneratorTest.java | 59 +
.../configuration/ConfigurationValidatorTest.java | 54 +
.../LoggingConfigurationFactoryTest.java | 89 ++
.../com/epam/datalab/core/BillingUtilsTest.java | 107 ++
.../datalab/core/aggregate/DataAggregatorTest.java | 47 +
.../datalab/core/aggregate/UsageDataListTest.java | 116 ++
.../core/parser/BillingResourceTypeTest.java | 36 +
.../epam/datalab/core/parser/ColumnInfoTest.java | 37 +
.../epam/datalab/core/parser/ColumnMetaTest.java | 86 ++
.../epam/datalab/core/parser/CommonFormatTest.java | 107 ++
.../datalab/core/parser/ConditionEvaluateTest.java | 118 ++
.../epam/datalab/core/parser/ReportLineTest.java | 94 ++
.../epam/datalab/logging/AppenderConsoleTest.java | 33 +
.../com/epam/datalab/logging/AppenderFileTest.java | 42 +
.../epam/datalab/module/AdapterConsoleTest.java | 33 +
.../com/epam/datalab/module/AdapterFileTest.java | 41 +
.../epam/datalab/module/AdapterMongoDBTest.java | 55 +
.../com/epam/datalab/module/ParserCsvTest.java | 81 ++
.../epam/datalab/module/aws/AdapterS3FileTest.java | 50 +
.../com/epam/datalab/module/aws/FilterAWSTest.java | 50 +
.../epam/datalab/module/aws/S3FileListTest.java | 125 +++
.../BillingToolConfigurationTest.java | 98 --
.../configuration/ConfigJsonGeneratorTest.java | 60 --
.../configuration/ConfigurationValidatorTest.java | 55 -
.../LoggingConfigurationFactoryTest.java | 91 --
.../java/com/epam/dlab/core/BillingUtilsTest.java | 108 --
.../dlab/core/aggregate/DataAggregatorTest.java | 47 -
.../dlab/core/aggregate/UsageDataListTest.java | 116 --
.../dlab/core/parser/BillingResourceTypeTest.java | 36 -
.../com/epam/dlab/core/parser/ColumnInfoTest.java | 37 -
.../com/epam/dlab/core/parser/ColumnMetaTest.java | 86 --
.../epam/dlab/core/parser/CommonFormatTest.java | 107 --
.../dlab/core/parser/ConditionEvaluateTest.java | 119 ---
.../com/epam/dlab/core/parser/ReportLineTest.java | 94 --
.../com/epam/dlab/logging/AppenderConsoleTest.java | 33 -
.../com/epam/dlab/logging/AppenderFileTest.java | 42 -
.../com/epam/dlab/module/AdapterConsoleTest.java | 33 -
.../java/com/epam/dlab/module/AdapterFileTest.java | 42 -
.../com/epam/dlab/module/AdapterMongoDBTest.java | 56 -
.../java/com/epam/dlab/module/ParserCsvTest.java | 82 --
.../epam/dlab/module/aws/AdapterS3FileTest.java | 51 -
.../com/epam/dlab/module/aws/FilterAWSTest.java | 50 -
.../com/epam/dlab/module/aws/S3FileListTest.java | 126 ---
services/billing-azure/billing.yml | 10 +-
services/billing-azure/pom.xml | 8 +-
.../azure/AzureInvoiceCalculationService.java | 242 +++++
.../billing/azure/BillingAzureApplication.java | 36 +
.../billing/azure/CalculateBillingService.java | 29 +
.../billing/azure/CalculateBillingServiceImpl.java | 245 +++++
.../billing/azure/MongoDbBillingClient.java | 50 +
.../epam/datalab/billing/azure/MongoDocument.java | 70 ++
.../config/AggregationOutputMongoDataSource.java | 31 +
.../azure/config/BillingConfigurationAzure.java | 52 +
.../azure/config/LoggingConfigurationFactory.java | 166 +++
.../billing/azure/config/SecurityConfig.java | 66 ++
.../azure/controller/BillingController.java | 44 +
.../billing/azure/logging/AppenderBase.java | 103 ++
.../billing/azure/logging/AppenderConsole.java | 44 +
.../billing/azure/logging/AppenderFile.java | 200 ++++
.../azure/model/AzureDailyResourceInvoice.java | 48 +
.../azure/model/AzureDatalabBillableResource.java | 39 +
.../datalab/billing/azure/model/BillingPeriod.java | 35 +
.../billing/azure/rate/AzureRateCardClient.java | 109 ++
.../com/epam/datalab/billing/azure/rate/Meter.java | 52 +
.../billing/azure/rate/RateCardResponse.java | 41 +
.../azure/usage/AzureUsageAggregateClient.java | 103 ++
.../datalab/billing/azure/usage/InstanceData.java | 29 +
.../billing/azure/usage/MicrosoftResources.java | 37 +
.../billing/azure/usage/UsageAggregateRecord.java | 35 +
.../usage/UsageAggregateRecordProperties.java | 54 +
.../azure/usage/UsageAggregateResponse.java | 33 +
.../azure/AzureInvoiceCalculationService.java | 242 -----
.../billing/azure/BillingAzureApplication.java | 36 -
.../billing/azure/CalculateBillingService.java | 29 -
.../billing/azure/CalculateBillingServiceImpl.java | 245 -----
.../dlab/billing/azure/MongoDbBillingClient.java | 50 -
.../com/epam/dlab/billing/azure/MongoDocument.java | 70 --
.../config/AggregationOutputMongoDataSource.java | 31 -
.../azure/config/BillingConfigurationAzure.java | 52 -
.../azure/config/LoggingConfigurationFactory.java | 166 ---
.../dlab/billing/azure/config/SecurityConfig.java | 66 --
.../azure/controller/BillingController.java | 44 -
.../dlab/billing/azure/logging/AppenderBase.java | 92 --
.../billing/azure/logging/AppenderConsole.java | 43 -
.../dlab/billing/azure/logging/AppenderFile.java | 173 ---
.../azure/model/AzureDailyResourceInvoice.java | 48 -
.../azure/model/AzureDlabBillableResource.java | 39 -
.../dlab/billing/azure/model/BillingPeriod.java | 35 -
.../billing/azure/rate/AzureRateCardClient.java | 105 --
.../com/epam/dlab/billing/azure/rate/Meter.java | 52 -
.../dlab/billing/azure/rate/RateCardResponse.java | 41 -
.../azure/usage/AzureUsageAggregateClient.java | 103 --
.../dlab/billing/azure/usage/InstanceData.java | 29 -
.../billing/azure/usage/MicrosoftResources.java | 37 -
.../billing/azure/usage/UsageAggregateRecord.java | 35 -
.../usage/UsageAggregateRecordProperties.java | 54 -
.../azure/usage/UsageAggregateResponse.java | 33 -
.../src/main/resources/application.yml | 18 +-
services/billing-gcp/billing.yml | 8 +-
services/billing-gcp/pom.xml | 8 +-
.../datalab/billing/gcp/BillingGcpApplication.java | 36 +
.../gcp/conf/BillingApplicationConfiguration.java | 34 +
.../billing/gcp/conf/DatalabConfiguration.java | 35 +
.../datalab/billing/gcp/conf/SecurityConfig.java | 66 ++
.../billing/gcp/controller/BillingController.java | 44 +
.../epam/datalab/billing/gcp/dao/BillingDAO.java | 29 +
.../billing/gcp/dao/impl/BigQueryBillingDAO.java | 131 +++
.../datalab/billing/gcp/model/BillingHistory.java | 32 +
.../gcp/repository/BillingHistoryRepository.java | 26 +
.../billing/gcp/service/BillingService.java | 29 +
.../gcp/service/impl/BillingServiceImpl.java | 52 +
.../dlab/billing/gcp/BillingGcpApplication.java | 36 -
.../gcp/conf/BillingApplicationConfiguration.java | 34 -
.../dlab/billing/gcp/conf/DlabConfiguration.java | 35 -
.../epam/dlab/billing/gcp/conf/SecurityConfig.java | 66 --
.../billing/gcp/controller/BillingController.java | 44 -
.../com/epam/dlab/billing/gcp/dao/BillingDAO.java | 29 -
.../billing/gcp/dao/impl/BigQueryBillingDAO.java | 131 ---
.../dlab/billing/gcp/model/BillingHistory.java | 32 -
.../dlab/billing/gcp/model/GcpBillingData.java | 44 -
.../gcp/repository/BillingHistoryRepository.java | 26 -
.../billing/gcp/repository/BillingRepository.java | 27 -
.../dlab/billing/gcp/service/BillingService.java | 29 -
.../gcp/service/impl/BillingServiceImpl.java | 52 -
.../billing-gcp/src/main/resources/application.yml | 16 +-
services/common/pom.xml | 4 +-
.../datalab/billing/BillingCalculationUtils.java | 42 +
.../epam/datalab/billing/DatalabResourceType.java | 54 +
.../epam/datalab/exceptions/AdapterException.java | 56 +
.../exceptions/DatalabAuthenticationException.java | 26 +
.../epam/datalab/exceptions/DatalabException.java | 34 +
.../exceptions/DatalabValidationException.java | 26 +
.../epam/datalab/exceptions/GenericException.java | 56 +
.../exceptions/InitializationException.java | 56 +
.../epam/datalab/exceptions/ParseException.java | 57 +
.../exceptions/ResourceAlreadyExistException.java | 26 +
.../exceptions/ResourceConflictException.java | 26 +
.../ResourceInappropriateStateException.java | 26 +
.../exceptions/ResourceNotFoundException.java | 26 +
.../exceptions/ResourceQuoteReachedException.java | 29 +
.../epam/dlab/billing/BillingCalculationUtils.java | 42 -
.../com/epam/dlab/billing/DlabResourceType.java | 54 -
.../com/epam/dlab/exceptions/AdapterException.java | 49 -
.../exceptions/DlabAuthenticationException.java | 26 -
.../com/epam/dlab/exceptions/DlabException.java | 34 -
.../dlab/exceptions/DlabValidationException.java | 26 -
.../com/epam/dlab/exceptions/GenericException.java | 49 -
.../dlab/exceptions/InitializationException.java | 49 -
.../com/epam/dlab/exceptions/ParseException.java | 50 -
.../exceptions/ResourceAlreadyExistException.java | 26 -
.../dlab/exceptions/ResourceConflictException.java | 26 -
.../ResourceInappropriateStateException.java | 26 -
.../dlab/exceptions/ResourceNotFoundException.java | 26 -
.../exceptions/ResourceQuoteReachedException.java | 29 -
services/datalab-model/pom.xml | 79 ++
.../main/java/com/epam/datalab/MongoKeyWords.java | 34 +
.../datalab/dto/InfrastructureMetaInfoDTO.java | 34 +
.../epam/datalab/dto/LibListComputationalDTO.java | 55 +
.../epam/datalab/dto/LibListExploratoryDTO.java | 40 +
.../java/com/epam/datalab/dto/LibraryGroups.java | 40 +
.../java/com/epam/datalab/dto/ResourceBaseDTO.java | 74 ++
.../com/epam/datalab/dto/ResourceEnvBaseDTO.java | 71 ++
.../com/epam/datalab/dto/ResourceSysBaseDTO.java | 87 ++
.../java/com/epam/datalab/dto/ResourceURL.java | 38 +
.../java/com/epam/datalab/dto/SchedulerJobDTO.java | 86 ++
.../java/com/epam/datalab/dto/StatusBaseDTO.java | 127 +++
.../com/epam/datalab/dto/StatusEnvBaseDTO.java | 117 ++
.../epam/datalab/dto/UserEnvironmentResources.java | 62 ++
.../java/com/epam/datalab/dto/UserInstanceDTO.java | 192 ++++
.../com/epam/datalab/dto/UserInstanceStatus.java | 67 ++
.../com/epam/datalab/dto/aws/AwsCloudSettings.java | 102 ++
.../computational/AwsComputationalResource.java | 78 ++
.../AwsComputationalTerminateDTO.java | 33 +
.../dto/aws/computational/ClusterConfig.java | 41 +
.../aws/computational/ComputationalConfigAws.java | 53 +
.../aws/computational/ComputationalCreateAws.java | 165 +++
.../computational/SparkComputationalConfigAws.java | 48 +
.../computational/SparkComputationalCreateAws.java | 78 ++
.../epam/datalab/dto/aws/edge/EdgeCreateAws.java | 53 +
.../com/epam/datalab/dto/aws/edge/EdgeInfoAws.java | 42 +
.../dto/aws/exploratory/ExploratoryCreateAws.java | 48 +
.../datalab/dto/aws/keyload/UploadFileAws.java | 35 +
.../epam/datalab/dto/azure/AzureCloudSettings.java | 96 ++
.../azure/auth/AuthorizationCodeFlowResponse.java | 63 ++
.../SparkComputationalConfigAzure.java | 46 +
.../SparkComputationalCreateAzure.java | 94 ++
.../datalab/dto/azure/edge/EdgeCreateAzure.java | 54 +
.../epam/datalab/dto/azure/edge/EdgeInfoAzure.java | 52 +
.../exploratory/ExploratoryActionStartAzure.java | 64 ++
.../exploratory/ExploratoryActionStopAzure.java | 44 +
.../azure/exploratory/ExploratoryCreateAzure.java | 85 ++
.../datalab/dto/azure/keyload/UploadFileAzure.java | 35 +
.../com/epam/datalab/dto/backup/EnvBackupDTO.java | 40 +
.../epam/datalab/dto/backup/EnvBackupStatus.java | 49 +
.../datalab/dto/backup/EnvBackupStatusDTO.java | 56 +
.../com/epam/datalab/dto/base/CloudSettings.java | 32 +
.../com/epam/datalab/dto/base/DataEngineType.java | 66 ++
.../dto/base/computational/ComputationalBase.java | 119 +++
.../computational/FullComputationalTemplate.java | 33 +
.../com/epam/datalab/dto/base/edge/EdgeInfo.java | 71 ++
.../datalab/dto/base/keyload/ReuploadFile.java | 33 +
.../epam/datalab/dto/base/keyload/UploadFile.java | 33 +
.../datalab/dto/base/keyload/UploadFileResult.java | 46 +
.../datalab/dto/base/project/ProjectResult.java | 37 +
.../com/epam/datalab/dto/billing/BillingData.java | 44 +
.../datalab/dto/billing/BillingResourceType.java | 31 +
.../com/epam/datalab/dto/bucket/BucketDTO.java | 34 +
.../epam/datalab/dto/bucket/BucketDeleteDTO.java | 32 +
.../epam/datalab/dto/bucket/FolderUploadDTO.java | 33 +
.../computational/CheckInactivityCallbackDTO.java | 57 +
.../dto/computational/CheckInactivityStatus.java | 49 +
.../computational/CheckInactivityStatusDTO.java | 30 +
.../ComputationalCheckInactivityDTO.java | 57 +
.../ComputationalClusterConfigDTO.java | 40 +
.../dto/computational/ComputationalStartDTO.java | 25 +
.../dto/computational/ComputationalStatusDTO.java | 118 ++
.../dto/computational/ComputationalStopDTO.java | 25 +
.../computational/ComputationalTerminateDTO.java | 25 +
.../SparkStandaloneClusterResource.java | 62 ++
.../computational/UserComputationalResource.java | 99 ++
.../dto/exploratory/ExploratoryActionDTO.java | 71 ++
.../dto/exploratory/ExploratoryBaseDTO.java | 85 ++
.../ExploratoryCheckInactivityAction.java | 23 +
.../dto/exploratory/ExploratoryCreateDTO.java | 134 +++
.../dto/exploratory/ExploratoryGitCreds.java | 193 ++++
.../dto/exploratory/ExploratoryGitCredsDTO.java | 63 ++
.../exploratory/ExploratoryGitCredsUpdateDTO.java | 62 ++
.../dto/exploratory/ExploratoryImageDTO.java | 72 ++
...xploratoryReconfigureSparkClusterActionDTO.java | 43 +
.../dto/exploratory/ExploratoryStatusDTO.java | 128 +++
.../dto/exploratory/ImageCreateStatusDTO.java | 74 ++
.../epam/datalab/dto/exploratory/ImageStatus.java | 39 +
.../datalab/dto/exploratory/LibInstallDTO.java | 85 ++
.../dto/exploratory/LibInstallStatusDTO.java | 57 +
.../datalab/dto/exploratory/LibListStatusDTO.java | 63 ++
.../epam/datalab/dto/exploratory/LibStatus.java | 49 +
.../datalab/dto/exploratory/LibraryInstallDTO.java | 67 ++
.../com/epam/datalab/dto/gcp/GcpCloudSettings.java | 96 ++
.../auth/GcpOauth2AuthorizationCodeResponse.java | 29 +
.../gcp/computational/ComputationalCreateGcp.java | 100 ++
.../computational/GcpComputationalResource.java | 77 ++
.../GcpComputationalTerminateDTO.java | 33 +
.../computational/SparkComputationalCreateGcp.java | 81 ++
.../epam/datalab/dto/gcp/edge/EdgeCreateGcp.java | 25 +
.../com/epam/datalab/dto/gcp/edge/EdgeInfoGcp.java | 40 +
.../dto/gcp/exploratory/ExploratoryCreateGcp.java | 48 +
.../datalab/dto/gcp/keyload/UploadFileGcp.java | 42 +
.../datalab/dto/imagemetadata/ApplicationDto.java | 85 ++
.../imagemetadata/ComputationalMetadataDTO.java | 53 +
.../ComputationalResourceShapeDto.java | 114 ++
.../ExploratoryEnvironmentVersion.java | 94 ++
.../dto/imagemetadata/ExploratoryMetadataDTO.java | 48 +
.../dto/imagemetadata/ImageMetadataDTO.java | 51 +
.../epam/datalab/dto/imagemetadata/ImageType.java | 46 +
.../datalab/dto/imagemetadata/TemplateDTO.java | 84 ++
.../epam/datalab/dto/keyload/KeyLoadStatus.java | 60 ++
.../com/epam/datalab/dto/keyload/UserKeyDTO.java | 58 +
.../epam/datalab/dto/project/ProjectActionDTO.java | 34 +
.../epam/datalab/dto/project/ProjectCreateDTO.java | 37 +
.../dto/reuploadkey/ReuploadKeyCallbackDTO.java | 64 ++
.../datalab/dto/reuploadkey/ReuploadKeyDTO.java | 56 +
.../datalab/dto/reuploadkey/ReuploadKeyStatus.java | 49 +
.../dto/reuploadkey/ReuploadKeyStatusDTO.java | 48 +
.../com/epam/datalab/dto/status/EnvResource.java | 161 +++
.../epam/datalab/dto/status/EnvResourceList.java | 91 ++
.../com/epam/datalab/dto/status/EnvStatusDTO.java | 65 ++
.../java/com/epam/datalab/model/ResourceData.java | 60 ++
.../java/com/epam/datalab/model/ResourceEnum.java | 38 +
.../java/com/epam/datalab/model/ResourceType.java | 37 +
.../java/com/epam/datalab/model/StringList.java | 47 +
.../datalab/model/aws/BillingResourceType.java | 80 ++
.../com/epam/datalab/model/aws/ReportLine.java | 233 ++++
.../epam/datalab/model/azure/AzureAuthFile.java | 39 +
.../datalab/model/exploratory/Exploratory.java | 41 +
.../com/epam/datalab/model/exploratory/Image.java | 46 +
.../com/epam/datalab/model/library/Library.java | 55 +
.../datalab/model/scheduler/SchedulerJobData.java | 44 +
.../epam/datalab/model/systeminfo/DiskInfo.java | 36 +
.../epam/datalab/model/systeminfo/MemoryInfo.java | 39 +
.../com/epam/datalab/model/systeminfo/OsInfo.java | 38 +
.../datalab/model/systeminfo/ProcessorInfo.java | 46 +
.../datalab/util/CloudSettingsDeserializer.java | 88 ++
.../java/com/epam/datalab/util/JsonGenerator.java | 54 +
.../datalab/dto/status/EnvResourceDTOTest.java | 80 ++
.../epam/datalab/dto/status/EnvStatusDTOTest.java | 89 ++
.../com/epam/datalab/util/JsonGeneratorTest.java | 37 +
services/datalab-mongo-migration/pom.xml | 39 +
.../com/epam/datalab/migration/DbMigration.java | 26 +
.../exception/DatalabDbMigrationException.java | 27 +
.../migration/mongo/DatalabMongoMigration.java | 48 +
.../mongo/changelog/DatalabChangeLog.java | 66 ++
services/datalab-utils/pom.xml | 52 +
.../main/java/com/epam/datalab/util/FileUtils.java | 61 ++
.../java/com/epam/datalab/util/SecurityUtils.java | 38 +
.../java/com/epam/datalab/util/ServiceUtils.java | 151 +++
.../java/com/epam/datalab/util/UsernameUtils.java | 36 +
.../datalab/util/mongo/IsoDateDeSerializer.java | 61 ++
.../epam/datalab/util/mongo/IsoDateSerializer.java | 48 +
.../util/mongo/IsoLocalDateDeSerializer.java | 47 +
.../datalab/util/mongo/IsoLocalDateSerializer.java | 38 +
.../util/mongo/IsoLocalDateTimeDeSerializer.java | 48 +
.../util/mongo/IsoLocalDateTimeSerializer.java | 38 +
.../epam/datalab/util/mongo/LongDeSerializer.java | 44 +
.../util/mongo/MongoStringDeserializer.java | 32 +
.../datalab/util/mongo/MongoStringSerializaer.java | 38 +
.../datalab/util/mongo/modules/IsoDateModule.java | 64 ++
.../util/mongo/modules/JavaPrimitiveModule.java | 30 +
.../datalab/util/mongo/modules/MongoModule.java | 31 +
.../com/epam/datalab/util/SecurityUtilsTest.java | 35 +
.../mongo/IsoLocalDateTimeDeSerializerTest.java | 89 ++
.../util/mongo/IsoLocalDateTimeSerDeTest.java | 70 ++
.../util/mongo/IsoLocalDateTimeSerializerTest.java | 57 +
services/datalab-webapp-common/pom.xml | 69 ++
.../src/main/java/com/epam/datalab/ModuleBase.java | 48 +
.../com/epam/datalab/ServiceConfiguration.java | 114 ++
.../datalab/auth/SecurityUnauthorizedHandler.java | 32 +
.../main/java/com/epam/datalab/auth/UserInfo.java | 222 ++++
.../epam/datalab/auth/contract/SecurityAPI.java | 30 +
.../epam/datalab/auth/dto/UserCredentialDTO.java | 74 ++
.../auth/oauth2/Oauth2AuthenticationService.java | 38 +
.../auth/rest/AbstractAuthenticationService.java | 47 +
.../datalab/auth/rest/ConfigurableResource.java | 36 +
.../auth/rest/UserSessionDurationAuthorizer.java | 90 ++
.../auth/rest/UserSessionDurationCallback.java | 27 +
.../java/com/epam/datalab/cloud/CloudModule.java | 34 +
.../java/com/epam/datalab/cloud/CloudProvider.java | 28 +
.../com/epam/datalab/constants/ServiceConsts.java | 34 +
.../java/com/epam/datalab/mongo/MongoService.java | 80 ++
.../epam/datalab/mongo/MongoServiceFactory.java | 93 ++
.../com/epam/datalab/rest/client/RESTService.java | 165 +++
.../datalab/rest/client/RESTServiceFactory.java | 61 ++
.../epam/datalab/rest/contracts/ApiCallbacks.java | 45 +
.../com/epam/datalab/rest/contracts/BackupAPI.java | 27 +
.../datalab/rest/contracts/ComputationalAPI.java | 41 +
.../com/epam/datalab/rest/contracts/DockerAPI.java | 27 +
.../datalab/rest/contracts/ExploratoryAPI.java | 34 +
.../datalab/rest/contracts/InfrasctructureAPI.java | 29 +
.../com/epam/datalab/rest/contracts/KeyAPI.java | 29 +
.../java/com/epam/datalab/rest/dto/ErrorDTO.java | 35 +
.../mappers/AuthenticationExceptionMapper.java | 38 +
.../mappers/DatalabValidationExceptionMapper.java | 38 +
.../rest/mappers/GenericExceptionMapper.java | 43 +
.../mappers/JsonProcessingExceptionMapper.java | 28 +
.../mappers/ResourceConflictExceptionMapper.java | 40 +
.../mappers/ResourceNotFoundExceptionMapper.java | 40 +
.../ResourceQuoteReachedExceptionMapper.java | 37 +
.../rest/mappers/RuntimeExceptionMapper.java | 63 ++
.../rest/mappers/ValidationExceptionMapper.java | 45 +
.../com/epam/datalab/validation/AwsValidation.java | 23 +
.../epam/datalab/validation/AzureValidation.java | 23 +
.../CloudConfigurationSequenceProvider.java | 66 ++
.../com/epam/datalab/validation/GcpValidation.java | 23 +
services/dlab-model/pom.xml | 80 --
.../src/main/java/com/epam/dlab/MongoKeyWords.java | 34 -
.../epam/dlab/dto/InfrastructureMetaInfoDTO.java | 34 -
.../com/epam/dlab/dto/LibListComputationalDTO.java | 55 -
.../com/epam/dlab/dto/LibListExploratoryDTO.java | 40 -
.../main/java/com/epam/dlab/dto/LibraryGroups.java | 40 -
.../java/com/epam/dlab/dto/ResourceBaseDTO.java | 74 --
.../java/com/epam/dlab/dto/ResourceEnvBaseDTO.java | 71 --
.../java/com/epam/dlab/dto/ResourceSysBaseDTO.java | 87 --
.../main/java/com/epam/dlab/dto/ResourceURL.java | 38 -
.../java/com/epam/dlab/dto/SchedulerJobDTO.java | 82 --
.../main/java/com/epam/dlab/dto/StatusBaseDTO.java | 127 ---
.../java/com/epam/dlab/dto/StatusEnvBaseDTO.java | 117 --
.../epam/dlab/dto/UserEnvironmentResources.java | 62 --
.../java/com/epam/dlab/dto/UserInstanceDTO.java | 192 ----
.../java/com/epam/dlab/dto/UserInstanceStatus.java | 67 --
.../com/epam/dlab/dto/aws/AwsCloudSettings.java | 97 --
.../computational/AwsComputationalResource.java | 78 --
.../AwsComputationalTerminateDTO.java | 33 -
.../dlab/dto/aws/computational/ClusterConfig.java | 41 -
.../aws/computational/ComputationalConfigAws.java | 53 -
.../aws/computational/ComputationalCreateAws.java | 165 ---
.../computational/SparkComputationalConfigAws.java | 48 -
.../computational/SparkComputationalCreateAws.java | 78 --
.../com/epam/dlab/dto/aws/edge/EdgeCreateAws.java | 53 -
.../com/epam/dlab/dto/aws/edge/EdgeInfoAws.java | 42 -
.../dto/aws/exploratory/ExploratoryCreateAws.java | 48 -
.../epam/dlab/dto/aws/keyload/UploadFileAws.java | 35 -
.../epam/dlab/dto/azure/AzureCloudSettings.java | 91 --
.../azure/auth/AuthorizationCodeFlowResponse.java | 63 --
.../SparkComputationalConfigAzure.java | 46 -
.../SparkComputationalCreateAzure.java | 94 --
.../epam/dlab/dto/azure/edge/EdgeCreateAzure.java | 54 -
.../epam/dlab/dto/azure/edge/EdgeInfoAzure.java | 52 -
.../exploratory/ExploratoryActionStartAzure.java | 64 --
.../exploratory/ExploratoryActionStopAzure.java | 44 -
.../azure/exploratory/ExploratoryCreateAzure.java | 85 --
.../dlab/dto/azure/keyload/UploadFileAzure.java | 35 -
.../com/epam/dlab/dto/backup/EnvBackupDTO.java | 40 -
.../com/epam/dlab/dto/backup/EnvBackupStatus.java | 49 -
.../epam/dlab/dto/backup/EnvBackupStatusDTO.java | 56 -
.../java/com/epam/dlab/dto/base/CloudSettings.java | 32 -
.../com/epam/dlab/dto/base/DataEngineType.java | 66 --
.../dto/base/computational/ComputationalBase.java | 119 ---
.../computational/FullComputationalTemplate.java | 33 -
.../java/com/epam/dlab/dto/base/edge/EdgeInfo.java | 71 --
.../epam/dlab/dto/base/keyload/ReuploadFile.java | 33 -
.../com/epam/dlab/dto/base/keyload/UploadFile.java | 33 -
.../dlab/dto/base/keyload/UploadFileResult.java | 46 -
.../epam/dlab/dto/base/project/ProjectResult.java | 37 -
.../com/epam/dlab/dto/billing/BillingData.java | 44 -
.../epam/dlab/dto/billing/BillingResourceType.java | 31 -
.../java/com/epam/dlab/dto/bucket/BucketDTO.java | 34 -
.../com/epam/dlab/dto/bucket/BucketDeleteDTO.java | 32 -
.../com/epam/dlab/dto/bucket/FolderUploadDTO.java | 33 -
.../computational/CheckInactivityCallbackDTO.java | 57 -
.../dto/computational/CheckInactivityStatus.java | 49 -
.../computational/CheckInactivityStatusDTO.java | 30 -
.../ComputationalCheckInactivityDTO.java | 57 -
.../ComputationalClusterConfigDTO.java | 40 -
.../dto/computational/ComputationalStartDTO.java | 25 -
.../dto/computational/ComputationalStatusDTO.java | 118 --
.../dto/computational/ComputationalStopDTO.java | 25 -
.../computational/ComputationalTerminateDTO.java | 25 -
.../SparkStandaloneClusterResource.java | 62 --
.../computational/UserComputationalResource.java | 99 --
.../dlab/dto/exploratory/ExploratoryActionDTO.java | 71 --
.../dlab/dto/exploratory/ExploratoryBaseDTO.java | 84 --
.../ExploratoryCheckInactivityAction.java | 23 -
.../dlab/dto/exploratory/ExploratoryCreateDTO.java | 134 ---
.../dlab/dto/exploratory/ExploratoryGitCreds.java | 193 ----
.../dto/exploratory/ExploratoryGitCredsDTO.java | 60 --
.../exploratory/ExploratoryGitCredsUpdateDTO.java | 58 -
.../dlab/dto/exploratory/ExploratoryImageDTO.java | 72 --
...xploratoryReconfigureSparkClusterActionDTO.java | 43 -
.../dlab/dto/exploratory/ExploratoryStatusDTO.java | 128 ---
.../dlab/dto/exploratory/ImageCreateStatusDTO.java | 74 --
.../com/epam/dlab/dto/exploratory/ImageStatus.java | 39 -
.../epam/dlab/dto/exploratory/LibInstallDTO.java | 85 --
.../dlab/dto/exploratory/LibInstallStatusDTO.java | 57 -
.../dlab/dto/exploratory/LibListStatusDTO.java | 63 --
.../com/epam/dlab/dto/exploratory/LibStatus.java | 49 -
.../dlab/dto/exploratory/LibraryInstallDTO.java | 67 --
.../com/epam/dlab/dto/gcp/GcpCloudSettings.java | 91 --
.../auth/GcpOauth2AuthorizationCodeResponse.java | 29 -
.../gcp/computational/ComputationalCreateGcp.java | 100 --
.../computational/GcpComputationalResource.java | 77 --
.../GcpComputationalTerminateDTO.java | 33 -
.../computational/SparkComputationalCreateGcp.java | 81 --
.../com/epam/dlab/dto/gcp/edge/EdgeCreateGcp.java | 25 -
.../com/epam/dlab/dto/gcp/edge/EdgeInfoGcp.java | 40 -
.../dto/gcp/exploratory/ExploratoryCreateGcp.java | 48 -
.../epam/dlab/dto/gcp/keyload/UploadFileGcp.java | 42 -
.../dlab/dto/imagemetadata/ApplicationDto.java | 85 --
.../imagemetadata/ComputationalMetadataDTO.java | 53 -
.../ComputationalResourceShapeDto.java | 114 --
.../ExploratoryEnvironmentVersion.java | 94 --
.../dto/imagemetadata/ExploratoryMetadataDTO.java | 48 -
.../dlab/dto/imagemetadata/ImageMetadataDTO.java | 51 -
.../com/epam/dlab/dto/imagemetadata/ImageType.java | 46 -
.../epam/dlab/dto/imagemetadata/TemplateDTO.java | 84 --
.../com/epam/dlab/dto/keyload/KeyLoadStatus.java | 60 --
.../java/com/epam/dlab/dto/keyload/UserKeyDTO.java | 58 -
.../epam/dlab/dto/project/ProjectActionDTO.java | 34 -
.../epam/dlab/dto/project/ProjectCreateDTO.java | 37 -
.../dto/reuploadkey/ReuploadKeyCallbackDTO.java | 64 --
.../epam/dlab/dto/reuploadkey/ReuploadKeyDTO.java | 56 -
.../dlab/dto/reuploadkey/ReuploadKeyStatus.java | 49 -
.../dlab/dto/reuploadkey/ReuploadKeyStatusDTO.java | 48 -
.../java/com/epam/dlab/dto/status/EnvResource.java | 161 ---
.../com/epam/dlab/dto/status/EnvResourceList.java | 78 --
.../com/epam/dlab/dto/status/EnvStatusDTO.java | 58 -
.../java/com/epam/dlab/model/ResourceData.java | 60 --
.../java/com/epam/dlab/model/ResourceEnum.java | 38 -
.../java/com/epam/dlab/model/ResourceType.java | 37 -
.../main/java/com/epam/dlab/model/StringList.java | 47 -
.../epam/dlab/model/aws/BillingResourceType.java | 75 --
.../java/com/epam/dlab/model/aws/ReportLine.java | 226 ----
.../com/epam/dlab/model/azure/AzureAuthFile.java | 39 -
.../epam/dlab/model/exploratory/Exploratory.java | 41 -
.../com/epam/dlab/model/exploratory/Image.java | 46 -
.../java/com/epam/dlab/model/library/Library.java | 55 -
.../dlab/model/scheduler/SchedulerJobData.java | 44 -
.../com/epam/dlab/model/systeminfo/DiskInfo.java | 36 -
.../com/epam/dlab/model/systeminfo/MemoryInfo.java | 39 -
.../com/epam/dlab/model/systeminfo/OsInfo.java | 38 -
.../epam/dlab/model/systeminfo/ProcessorInfo.java | 46 -
.../epam/dlab/util/CloudSettingsDeserializer.java | 88 --
.../java/com/epam/dlab/util/JsonGenerator.java | 54 -
.../epam/dlab/dto/status/EnvResourceDTOTest.java | 81 --
.../com/epam/dlab/dto/status/EnvStatusDTOTest.java | 89 --
.../java/com/epam/dlab/util/JsonGeneratorTest.java | 37 -
services/dlab-mongo-migration/pom.xml | 39 -
.../java/com/epam/dlab/migration/DbMigration.java | 26 -
.../exception/DlabDbMigrationException.java | 27 -
.../dlab/migration/mongo/DlabMongoMigration.java | 48 -
.../migration/mongo/changelog/DlabChangeLog.java | 66 --
services/dlab-utils/pom.xml | 52 -
.../main/java/com/epam/dlab/util/FileUtils.java | 61 --
.../java/com/epam/dlab/util/SecurityUtils.java | 38 -
.../main/java/com/epam/dlab/util/ServiceUtils.java | 141 ---
.../java/com/epam/dlab/util/UsernameUtils.java | 36 -
.../epam/dlab/util/mongo/IsoDateDeSerializer.java | 61 --
.../epam/dlab/util/mongo/IsoDateSerializer.java | 48 -
.../dlab/util/mongo/IsoLocalDateDeSerializer.java | 47 -
.../dlab/util/mongo/IsoLocalDateSerializer.java | 38 -
.../util/mongo/IsoLocalDateTimeDeSerializer.java | 48 -
.../util/mongo/IsoLocalDateTimeSerializer.java | 38 -
.../com/epam/dlab/util/mongo/LongDeSerializer.java | 44 -
.../dlab/util/mongo/MongoStringDeserializer.java | 32 -
.../dlab/util/mongo/MongoStringSerializaer.java | 38 -
.../dlab/util/mongo/modules/IsoDateModule.java | 59 -
.../util/mongo/modules/JavaPrimitiveModule.java | 30 -
.../epam/dlab/util/mongo/modules/MongoModule.java | 31 -
.../java/com/epam/dlab/util/SecurityUtilsTest.java | 35 -
.../mongo/IsoLocalDateTimeDeSerializerTest.java | 84 --
.../dlab/util/mongo/IsoLocalDateTimeSerDeTest.java | 70 --
.../util/mongo/IsoLocalDateTimeSerializerTest.java | 54 -
services/dlab-webapp-common/pom.xml | 69 --
.../src/main/java/com/epam/dlab/ModuleBase.java | 42 -
.../java/com/epam/dlab/ServiceConfiguration.java | 114 --
.../dlab/auth/SecurityUnauthorizedHandler.java | 32 -
.../src/main/java/com/epam/dlab/auth/UserInfo.java | 223 ----
.../com/epam/dlab/auth/contract/SecurityAPI.java | 30 -
.../com/epam/dlab/auth/dto/UserCredentialDTO.java | 74 --
.../auth/oauth2/Oauth2AuthenticationService.java | 38 -
.../auth/rest/AbstractAuthenticationService.java | 47 -
.../epam/dlab/auth/rest/ConfigurableResource.java | 36 -
.../auth/rest/UserSessionDurationAuthorizer.java | 90 --
.../auth/rest/UserSessionDurationCallback.java | 27 -
.../main/java/com/epam/dlab/cloud/CloudModule.java | 34 -
.../java/com/epam/dlab/cloud/CloudProvider.java | 28 -
.../com/epam/dlab/constants/ServiceConsts.java | 34 -
.../java/com/epam/dlab/mongo/MongoService.java | 80 --
.../com/epam/dlab/mongo/MongoServiceFactory.java | 93 --
.../com/epam/dlab/rest/client/RESTService.java | 165 ---
.../epam/dlab/rest/client/RESTServiceFactory.java | 61 --
.../com/epam/dlab/rest/contracts/ApiCallbacks.java | 45 -
.../com/epam/dlab/rest/contracts/BackupAPI.java | 26 -
.../epam/dlab/rest/contracts/ComputationalAPI.java | 41 -
.../com/epam/dlab/rest/contracts/DockerAPI.java | 27 -
.../epam/dlab/rest/contracts/ExploratoryAPI.java | 34 -
.../dlab/rest/contracts/InfrasctructureAPI.java | 29 -
.../java/com/epam/dlab/rest/contracts/KeyAPI.java | 29 -
.../main/java/com/epam/dlab/rest/dto/ErrorDTO.java | 35 -
.../mappers/AuthenticationExceptionMapper.java | 38 -
.../mappers/DlabValidationExceptionMapper.java | 38 -
.../dlab/rest/mappers/GenericExceptionMapper.java | 43 -
.../mappers/JsonProcessingExceptionMapper.java | 28 -
.../mappers/ResourceConflictExceptionMapper.java | 40 -
.../mappers/ResourceNotFoundExceptionMapper.java | 40 -
.../ResourceQuoteReachedExceptionMapper.java | 37 -
.../dlab/rest/mappers/RuntimeExceptionMapper.java | 63 --
.../rest/mappers/ValidationExceptionMapper.java | 45 -
.../com/epam/dlab/validation/AwsValidation.java | 23 -
.../com/epam/dlab/validation/AzureValidation.java | 23 -
.../CloudConfigurationSequenceProvider.java | 66 --
.../com/epam/dlab/validation/GcpValidation.java | 23 -
services/provisioning-service/pom.xml | 32 +-
services/provisioning-service/provisioning.yml | 14 +-
.../DropwizardBearerTokenFilterImpl.java | 39 +
.../backendapi/ProvisioningServiceApplication.java | 161 +++
...rovisioningServiceApplicationConfiguration.java | 197 ++++
.../backendapi/conf/CloudConfiguration.java | 83 ++
.../epam/datalab/backendapi/core/Constants.java | 29 +
.../epam/datalab/backendapi/core/Directories.java | 31 +
.../backendapi/core/DirectoriesCreator.java | 49 +
.../datalab/backendapi/core/DockerWarmuper.java | 172 +++
.../backendapi/core/FileHandlerCallback.java | 41 +
.../datalab/backendapi/core/MetadataHolder.java | 31 +
.../backendapi/core/commands/CmdCommand.java | 25 +
.../backendapi/core/commands/CommandBuilder.java | 186 ++++
.../backendapi/core/commands/CommandExecutor.java | 38 +
.../core/commands/CommandExecutorMock.java | 114 ++
.../core/commands/CommandExecutorMockAsync.java | 406 +++++++
.../core/commands/CommandParserMock.java | 377 +++++++
.../backendapi/core/commands/DockerAction.java | 54 +
.../backendapi/core/commands/DockerCommands.java | 69 ++
.../backendapi/core/commands/ICommandExecutor.java | 27 +
.../core/commands/ImagesDockerCommand.java | 53 +
.../core/commands/PythonBackupCommand.java | 81 ++
.../backendapi/core/commands/PythonCommand.java | 50 +
.../backendapi/core/commands/RunDockerCommand.java | 285 +++++
.../backendapi/core/commands/UnixCommand.java | 53 +
.../response/folderlistener/AsyncFileHandler.java | 135 +++
.../response/folderlistener/FolderListener.java | 434 ++++++++
.../folderlistener/FolderListenerExecutor.java | 55 +
.../core/response/folderlistener/WatchItem.java | 258 +++++
.../response/folderlistener/WatchItemList.java | 277 +++++
.../response/handlers/BackupCallbackHandler.java | 126 +++
.../handlers/CheckInactivityCallbackHandler.java | 141 +++
.../handlers/ComputationalCallbackHandler.java | 133 +++
.../response/handlers/ComputationalConfigure.java | 140 +++
.../ComputationalConfigureCallbackHandler.java | 63 ++
.../response/handlers/EdgeCallbackHandler.java | 78 ++
.../handlers/ExploratoryCallbackHandler.java | 108 ++
.../ExploratoryGitCredsCallbackHandler.java | 65 ++
.../handlers/ImageCreateCallbackHandler.java | 106 ++
.../handlers/LibInstallCallbackHandler.java | 118 ++
.../response/handlers/LibListCallbackHandler.java | 110 ++
.../response/handlers/PersistentFileHandler.java | 51 +
.../response/handlers/ProjectCallbackHandler.java | 71 ++
.../response/handlers/ResourceCallbackHandler.java | 210 ++++
.../handlers/ResourcesStatusCallbackHandler.java | 93 ++
.../handlers/ReuploadKeyCallbackHandler.java | 124 +++
.../response/handlers/dao/CallbackHandlerDao.java | 33 +
.../handlers/dao/FileSystemCallbackHandlerDao.java | 121 +++
.../backendapi/modules/AwsProvisioningModule.java | 39 +
.../modules/AzureProvisioningModule.java | 39 +
.../modules/CloudModuleConfigurator.java | 42 +
.../backendapi/modules/GcpProvisioningModule.java | 39 +
.../datalab/backendapi/modules/ModuleFactory.java | 44 +
.../backendapi/modules/ProductionModule.java | 89 ++
.../backendapi/modules/ProvisioningDevModule.java | 148 +++
.../backendapi/resources/BackupResource.java | 76 ++
.../backendapi/resources/BucketResource.java | 144 +++
.../resources/CallbackHandlerResource.java | 46 +
.../backendapi/resources/DockerResource.java | 92 ++
.../resources/GitExploratoryResource.java | 95 ++
.../backendapi/resources/ImageResource.java | 88 ++
.../resources/InfrastructureResource.java | 65 ++
.../backendapi/resources/LibraryResource.java | 195 ++++
.../backendapi/resources/ProjectResource.java | 76 ++
.../resources/ProvisioningHealthCheckResource.java | 44 +
.../resources/aws/ComputationalResourceAws.java | 197 ++++
.../backendapi/resources/aws/EdgeResourceAws.java | 94 ++
.../resources/aws/ExploratoryResourceAws.java | 77 ++
.../resources/aws/InfrastructureResourceAws.java | 49 +
.../azure/ComputationalResourceAzure.java | 94 ++
.../resources/azure/EdgeResourceAzure.java | 94 ++
.../resources/azure/ExploratoryResourceAzure.java | 75 ++
.../azure/InfrastructureResourceAzure.java | 50 +
.../backendapi/resources/base/EdgeService.java | 108 ++
.../resources/base/ExploratoryService.java | 79 ++
.../resources/base/InfrastructureService.java | 157 +++
.../backendapi/resources/base/KeyResource.java | 86 ++
.../resources/gcp/ComputationalResourceGcp.java | 197 ++++
.../backendapi/resources/gcp/EdgeResourceGcp.java | 96 ++
.../resources/gcp/ExploratoryResourceGcp.java | 77 ++
.../resources/gcp/InfrastructureResourceGcp.java | 49 +
.../datalab/backendapi/service/BucketService.java | 41 +
.../backendapi/service/CheckInactivityService.java | 29 +
.../datalab/backendapi/service/ProjectService.java | 35 +
.../service/RestoreCallbackHandlerService.java | 25 +
.../service/impl/CheckInactivityServiceImpl.java | 129 +++
.../backendapi/service/impl/DockerService.java | 42 +
.../backendapi/service/impl/KeyService.java | 146 +++
.../service/impl/ProjectServiceImpl.java | 135 +++
.../impl/RestoreCallbackHandlerServiceImpl.java | 58 +
.../service/impl/SparkClusterService.java | 156 +++
.../service/impl/aws/BucketServiceAwsImpl.java | 153 +++
.../service/impl/azure/BucketServiceAzureImpl.java | 177 +++
.../service/impl/gcp/BucketServiceGcpImpl.java | 132 +++
...gServiceCloudConfigurationSequenceProvider.java | 28 +
.../com/epam/datalab/process/ProcessConveyor.java | 59 +
.../process/builder/ProcessInfoBuilder.java | 288 +++++
.../process/exception/DatalabProcessException.java | 42 +
.../epam/datalab/process/model/DatalabProcess.java | 167 +++
.../com/epam/datalab/process/model/ProcessId.java | 70 ++
.../epam/datalab/process/model/ProcessInfo.java | 120 +++
.../epam/datalab/process/model/ProcessStatus.java | 32 +
.../epam/datalab/process/model/ProcessStep.java | 47 +
.../DropwizardBearerTokenFilterImpl.java | 39 -
.../backendapi/ProvisioningServiceApplication.java | 161 ---
...rovisioningServiceApplicationConfiguration.java | 197 ----
.../dlab/backendapi/conf/CloudConfiguration.java | 83 --
.../com/epam/dlab/backendapi/core/Constants.java | 29 -
.../com/epam/dlab/backendapi/core/Directories.java | 31 -
.../dlab/backendapi/core/DirectoriesCreator.java | 49 -
.../epam/dlab/backendapi/core/DockerWarmuper.java | 167 ---
.../dlab/backendapi/core/FileHandlerCallback.java | 41 -
.../epam/dlab/backendapi/core/MetadataHolder.java | 31 -
.../dlab/backendapi/core/commands/CmdCommand.java | 25 -
.../backendapi/core/commands/CommandBuilder.java | 186 ----
.../backendapi/core/commands/CommandExecutor.java | 38 -
.../core/commands/CommandExecutorMock.java | 114 --
.../core/commands/CommandExecutorMockAsync.java | 406 -------
.../core/commands/CommandParserMock.java | 373 -------
.../backendapi/core/commands/DockerAction.java | 54 -
.../backendapi/core/commands/DockerCommands.java | 69 --
.../backendapi/core/commands/ICommandExecutor.java | 26 -
.../core/commands/ImagesDockerCommand.java | 53 -
.../core/commands/PythonBackupCommand.java | 81 --
.../backendapi/core/commands/PythonCommand.java | 50 -
.../backendapi/core/commands/RunDockerCommand.java | 285 -----
.../dlab/backendapi/core/commands/UnixCommand.java | 53 -
.../response/folderlistener/AsyncFileHandler.java | 135 ---
.../response/folderlistener/FolderListener.java | 434 --------
.../folderlistener/FolderListenerExecutor.java | 55 -
.../core/response/folderlistener/WatchItem.java | 214 ----
.../response/folderlistener/WatchItemList.java | 277 -----
.../response/handlers/BackupCallbackHandler.java | 126 ---
.../handlers/CheckInactivityCallbackHandler.java | 141 ---
.../handlers/ComputationalCallbackHandler.java | 133 ---
.../response/handlers/ComputationalConfigure.java | 140 ---
.../ComputationalConfigureCallbackHandler.java | 63 --
.../response/handlers/EdgeCallbackHandler.java | 78 --
.../handlers/ExploratoryCallbackHandler.java | 108 --
.../ExploratoryGitCredsCallbackHandler.java | 65 --
.../handlers/ImageCreateCallbackHandler.java | 106 --
.../handlers/LibInstallCallbackHandler.java | 118 --
.../response/handlers/LibListCallbackHandler.java | 110 --
.../response/handlers/PersistentFileHandler.java | 51 -
.../response/handlers/ProjectCallbackHandler.java | 71 --
.../response/handlers/ResourceCallbackHandler.java | 210 ----
.../handlers/ResourcesStatusCallbackHandler.java | 93 --
.../handlers/ReuploadKeyCallbackHandler.java | 124 ---
.../response/handlers/dao/CallbackHandlerDao.java | 33 -
.../handlers/dao/FileSystemCallbackHandlerDao.java | 121 ---
.../backendapi/modules/AwsProvisioningModule.java | 39 -
.../modules/AzureProvisioningModule.java | 39 -
.../modules/CloudModuleConfigurator.java | 42 -
.../backendapi/modules/GcpProvisioningModule.java | 39 -
.../dlab/backendapi/modules/ModuleFactory.java | 44 -
.../dlab/backendapi/modules/ProductionModule.java | 89 --
.../backendapi/modules/ProvisioningDevModule.java | 148 ---
.../dlab/backendapi/resources/BackupResource.java | 76 --
.../dlab/backendapi/resources/BucketResource.java | 144 ---
.../resources/CallbackHandlerResource.java | 46 -
.../dlab/backendapi/resources/DockerResource.java | 87 --
.../resources/GitExploratoryResource.java | 95 --
.../dlab/backendapi/resources/ImageResource.java | 88 --
.../resources/InfrastructureResource.java | 61 --
.../dlab/backendapi/resources/LibraryResource.java | 195 ----
.../dlab/backendapi/resources/ProjectResource.java | 76 --
.../resources/ProvisioningHealthCheckResource.java | 44 -
.../resources/aws/ComputationalResourceAws.java | 197 ----
.../backendapi/resources/aws/EdgeResourceAws.java | 94 --
.../resources/aws/ExploratoryResourceAws.java | 77 --
.../resources/aws/InfrastructureResourceAws.java | 49 -
.../azure/ComputationalResourceAzure.java | 94 --
.../resources/azure/EdgeResourceAzure.java | 94 --
.../resources/azure/ExploratoryResourceAzure.java | 75 --
.../azure/InfrastructureResourceAzure.java | 50 -
.../backendapi/resources/base/EdgeService.java | 104 --
.../resources/base/ExploratoryService.java | 79 --
.../resources/base/InfrastructureService.java | 157 ---
.../backendapi/resources/base/KeyResource.java | 80 --
.../resources/gcp/ComputationalResourceGcp.java | 197 ----
.../backendapi/resources/gcp/EdgeResourceGcp.java | 96 --
.../resources/gcp/ExploratoryResourceGcp.java | 77 --
.../resources/gcp/InfrastructureResourceGcp.java | 49 -
.../dlab/backendapi/service/BucketService.java | 41 -
.../backendapi/service/CheckInactivityService.java | 29 -
.../dlab/backendapi/service/ProjectService.java | 35 -
.../service/RestoreCallbackHandlerService.java | 25 -
.../service/impl/CheckInactivityServiceImpl.java | 129 ---
.../backendapi/service/impl/DockerService.java | 42 -
.../dlab/backendapi/service/impl/KeyService.java | 146 ---
.../service/impl/ProjectServiceImpl.java | 135 ---
.../impl/RestoreCallbackHandlerServiceImpl.java | 58 -
.../service/impl/SparkClusterService.java | 156 ---
.../service/impl/aws/BucketServiceAwsImpl.java | 153 ---
.../service/impl/azure/BucketServiceAzureImpl.java | 177 ---
.../service/impl/gcp/BucketServiceGcpImpl.java | 132 ---
...gServiceCloudConfigurationSequenceProvider.java | 28 -
.../com/epam/dlab/process/ProcessConveyor.java | 59 -
.../dlab/process/builder/ProcessInfoBuilder.java | 288 -----
.../process/exception/DlabProcessException.java | 42 -
.../com/epam/dlab/process/model/DlabProcess.java | 167 ---
.../com/epam/dlab/process/model/ProcessId.java | 70 --
.../com/epam/dlab/process/model/ProcessInfo.java | 120 ---
.../com/epam/dlab/process/model/ProcessStatus.java | 32 -
.../com/epam/dlab/process/model/ProcessStep.java | 47 -
.../main/resources/mock_response/aws/backup.json | 2 +-
.../aws/dataengine-service_configure.json | 2 +-
.../aws/dataengine-service_configure_failed.json | 2 +-
.../aws/dataengine-service_create.json | 2 +-
.../aws/dataengine-service_create_failed.json | 2 +-
.../aws/dataengine-service_lib_install.json | 2 +-
.../aws/dataengine-service_lib_list.json | 4 +-
.../aws/dataengine-service_terminate.json | 2 +-
.../mock_response/aws/dataengine_configure.json | 2 +-
.../mock_response/aws/dataengine_create.json | 2 +-
.../mock_response/aws/dataengine_lib_install.json | 2 +-
.../mock_response/aws/dataengine_lib_list.json | 4 +-
.../mock_response/aws/dataengine_start.json | 2 +-
.../mock_response/aws/dataengine_stop.json | 2 +-
.../mock_response/aws/dataengine_terminate.json | 2 +-
.../resources/mock_response/aws/edge_create.json | 4 +-
.../resources/mock_response/aws/edge_start.json | 2 +-
.../resources/mock_response/aws/edge_stop.json | 2 +-
.../mock_response/aws/edge_terminate.json | 2 +-
.../aws/notebook_check_inactivity.json | 2 +-
.../mock_response/aws/notebook_create.json | 2 +-
.../mock_response/aws/notebook_create_failed.json | 2 +-
.../mock_response/aws/notebook_create_image.json | 2 +-
.../mock_response/aws/notebook_git_creds.json | 2 +-
.../mock_response/aws/notebook_lib_install.json | 6 +-
.../mock_response/aws/notebook_lib_list.json | 4 +-
.../mock_response/aws/notebook_start.json | 2 +-
.../mock_response/aws/notebook_status.json | 14 +-
.../resources/mock_response/aws/notebook_stop.json | 2 +-
.../mock_response/aws/notebook_terminate.json | 2 +-
.../mock_response/aws/project_create.json | 8 +-
.../mock_response/aws/project_terminate.json | 2 +-
.../main/resources/mock_response/azure/backup.json | 2 +-
.../azure/dataengine-service_configure.json | 2 +-
.../azure/dataengine-service_configure_failed.json | 2 +-
.../azure/dataengine-service_create.json | 2 +-
.../azure/dataengine-service_create_failed.json | 2 +-
.../azure/dataengine-service_lib_install.json | 2 +-
.../azure/dataengine-service_lib_list.json | 4 +-
.../azure/dataengine-service_terminate.json | 2 +-
.../mock_response/azure/dataengine_configure.json | 2 +-
.../mock_response/azure/dataengine_create.json | 2 +-
.../azure/dataengine_lib_install.json | 2 +-
.../mock_response/azure/dataengine_lib_list.json | 4 +-
.../mock_response/azure/dataengine_start.json | 2 +-
.../mock_response/azure/dataengine_stop.json | 2 +-
.../mock_response/azure/dataengine_terminate.json | 2 +-
.../resources/mock_response/azure/edge_create.json | 2 +-
.../resources/mock_response/azure/edge_start.json | 2 +-
.../resources/mock_response/azure/edge_stop.json | 2 +-
.../mock_response/azure/edge_terminate.json | 2 +-
.../mock_response/azure/notebook_create.json | 2 +-
.../azure/notebook_create_failed.json | 2 +-
.../mock_response/azure/notebook_create_image.json | 2 +-
.../mock_response/azure/notebook_git_creds.json | 2 +-
.../mock_response/azure/notebook_lib_install.json | 6 +-
.../mock_response/azure/notebook_lib_list.json | 4 +-
.../mock_response/azure/notebook_start.json | 2 +-
.../mock_response/azure/notebook_status.json | 14 +-
.../mock_response/azure/notebook_stop.json | 2 +-
.../mock_response/azure/notebook_terminate.json | 2 +-
.../mock_response/azure/project_create.json | 8 +-
.../mock_response/azure/project_terminate.json | 2 +-
.../main/resources/mock_response/gcp/backup.json | 2 +-
.../gcp/dataengine-service_configure.json | 2 +-
.../gcp/dataengine-service_create.json | 2 +-
.../gcp/dataengine-service_lib_install.json | 2 +-
.../gcp/dataengine-service_lib_list.json | 4 +-
.../gcp/dataengine-service_terminate.json | 2 +-
.../mock_response/gcp/dataengine_configure.json | 2 +-
.../mock_response/gcp/dataengine_create.json | 2 +-
.../mock_response/gcp/dataengine_lib_install.json | 2 +-
.../mock_response/gcp/dataengine_lib_list.json | 4 +-
.../mock_response/gcp/dataengine_start.json | 2 +-
.../mock_response/gcp/dataengine_stop.json | 2 +-
.../mock_response/gcp/dataengine_terminate.json | 2 +-
.../resources/mock_response/gcp/edge_create.json | 6 +-
.../resources/mock_response/gcp/edge_start.json | 2 +-
.../resources/mock_response/gcp/edge_stop.json | 2 +-
.../mock_response/gcp/edge_terminate.json | 2 +-
.../mock_response/gcp/notebook_create.json | 2 +-
.../mock_response/gcp/notebook_create_image.json | 2 +-
.../mock_response/gcp/notebook_git_creds.json | 2 +-
.../mock_response/gcp/notebook_lib_install.json | 6 +-
.../mock_response/gcp/notebook_lib_list.json | 4 +-
.../mock_response/gcp/notebook_start.json | 2 +-
.../mock_response/gcp/notebook_status.json | 14 +-
.../resources/mock_response/gcp/notebook_stop.json | 2 +-
.../mock_response/gcp/notebook_terminate.json | 2 +-
.../mock_response/gcp/project_create.json | 8 +-
.../mock_response/gcp/project_terminate.json | 2 +-
.../backendapi/core/DockerWarmuperTest.java | 133 +++
.../core/commands/CommandExecutorMockTest.java | 398 +++++++
.../docker/command/ImagesDockerCommandTest.java | 44 +
.../core/docker/command/RunDockerCommandTest.java | 319 ++++++
.../folderlistener/FolderListenerTest.java | 190 ++++
.../response/folderlistener/WatchItemListTest.java | 133 +++
.../response/folderlistener/WatchItemTest.java | 145 +++
.../dao/FileSystemCallbackHandlerDaoTest.java | 213 ++++
.../RestoreCallbackHandlerServiceImplTest.java | 65 ++
.../epam/datalab/rest/client/RESTServiceMock.java | 41 +
.../dlab/backendapi/core/DockerWarmuperTest.java | 129 ---
.../core/commands/CommandExecutorMockTest.java | 398 -------
.../docker/command/ImagesDockerCommandTest.java | 44 -
.../core/docker/command/RunDockerCommandTest.java | 319 ------
.../folderlistener/FolderListenerTest.java | 190 ----
.../response/folderlistener/WatchItemListTest.java | 133 ---
.../response/folderlistener/WatchItemTest.java | 145 ---
.../dao/FileSystemCallbackHandlerDaoTest.java | 208 ----
.../RestoreCallbackHandlerServiceImplTest.java | 63 --
.../com/epam/dlab/rest/client/RESTServiceMock.java | 41 -
services/self-service/entrypoint.sh | 2 +-
services/self-service/pom.xml | 56 +-
services/self-service/self-service.yml | 6 +-
.../datalab/backendapi/SelfServiceApplication.java | 217 ++++
.../epam/datalab/backendapi/annotation/Audit.java | 36 +
.../backendapi/annotation/BudgetLimited.java | 34 +
.../epam/datalab/backendapi/annotation/Info.java | 30 +
.../datalab/backendapi/annotation/Project.java | 31 +
.../backendapi/annotation/ProjectAdmin.java | 30 +
.../backendapi/annotation/ResourceName.java | 30 +
.../epam/datalab/backendapi/annotation/User.java | 30 +
.../backendapi/auth/KeycloakAuthenticator.java | 52 +
.../auth/SelfServiceSecurityAuthorizer.java | 34 +
.../filters/DropwizardBearerTokenFilterImpl.java | 39 +
.../backendapi/conf/CloudConfiguration.java | 59 +
.../backendapi/conf/KeycloakConfiguration.java | 27 +
.../conf/SelfServiceApplicationConfiguration.java | 264 +++++
.../com/epam/datalab/backendapi/dao/AuditDAO.java | 31 +
.../epam/datalab/backendapi/dao/AuditDAOImpl.java | 172 +++
.../com/epam/datalab/backendapi/dao/BackupDAO.java | 35 +
.../epam/datalab/backendapi/dao/BackupDAOImpl.java | 65 ++
.../datalab/backendapi/dao/BaseBillingDAO.java | 252 +++++
.../com/epam/datalab/backendapi/dao/BaseDAO.java | 526 +++++++++
.../epam/datalab/backendapi/dao/BillingDAO.java | 53 +
.../datalab/backendapi/dao/ComputationalDAO.java | 403 +++++++
.../com/epam/datalab/backendapi/dao/DockerDAO.java | 43 +
.../epam/datalab/backendapi/dao/EndpointDAO.java | 53 +
.../datalab/backendapi/dao/EndpointDAOImpl.java | 90 ++
.../com/epam/datalab/backendapi/dao/EnvDAO.java | 512 +++++++++
.../datalab/backendapi/dao/ExploratoryDAO.java | 479 +++++++++
.../datalab/backendapi/dao/ExploratoryLibDAO.java | 402 +++++++
.../epam/datalab/backendapi/dao/GitCredsDAO.java | 99 ++
.../backendapi/dao/ImageExploratoryDAO.java | 46 +
.../backendapi/dao/ImageExploratoryDAOImpl.java | 155 +++
.../epam/datalab/backendapi/dao/IndexCreator.java | 45 +
.../datalab/backendapi/dao/MongoCollections.java | 82 ++
.../epam/datalab/backendapi/dao/MongoSetting.java | 107 ++
.../epam/datalab/backendapi/dao/ProjectDAO.java | 59 +
.../datalab/backendapi/dao/ProjectDAOImpl.java | 190 ++++
.../epam/datalab/backendapi/dao/RequestIdDAO.java | 68 ++
.../datalab/backendapi/dao/SchedulerJobDAO.java | 248 +++++
.../epam/datalab/backendapi/dao/SecurityDAO.java | 118 ++
.../epam/datalab/backendapi/dao/SettingsDAO.java | 424 ++++++++
.../epam/datalab/backendapi/dao/UserGroupDAO.java | 33 +
.../datalab/backendapi/dao/UserGroupDAOImpl.java | 68 ++
.../epam/datalab/backendapi/dao/UserRoleDAO.java | 51 +
.../datalab/backendapi/dao/UserRoleDAOImpl.java | 215 ++++
.../datalab/backendapi/dao/UserSettingsDAO.java | 77 ++
.../datalab/backendapi/domain/AuditActionEnum.java | 24 +
.../datalab/backendapi/domain/AuditCreateDTO.java | 37 +
.../epam/datalab/backendapi/domain/AuditDTO.java | 39 +
.../backendapi/domain/AuditPaginationDTO.java | 43 +
.../backendapi/domain/AuditResourceTypeEnum.java | 24 +
.../backendapi/domain/AutoCompleteEnum.java | 24 +
.../datalab/backendapi/domain/BillingReport.java | 45 +
.../backendapi/domain/BillingReportLine.java | 56 +
.../epam/datalab/backendapi/domain/BudgetDTO.java | 36 +
.../backendapi/domain/CreateProjectDTO.java | 44 +
.../datalab/backendapi/domain/EndpointDTO.java | 50 +
.../backendapi/domain/EndpointResourcesDTO.java | 33 +
.../backendapi/domain/ExploratoryLibCache.java | 240 +++++
.../backendapi/domain/ExploratoryLibList.java | 250 +++++
.../domain/MavenSearchArtifactResponse.java | 82 ++
.../backendapi/domain/NotebookTemplate.java | 40 +
.../epam/datalab/backendapi/domain/ProjectDTO.java | 104 ++
.../backendapi/domain/ProjectEndpointDTO.java | 31 +
.../epam/datalab/backendapi/domain/RequestId.java | 95 ++
.../datalab/backendapi/domain/RequestIdDTO.java | 146 +++
.../domain/SchedulerConfigurationData.java | 31 +
.../backendapi/domain/UpdateProjectBudgetDTO.java | 35 +
.../backendapi/domain/UpdateProjectDTO.java | 40 +
.../dropwizard/bundles/DatalabKeycloakBundle.java | 58 +
.../dropwizard/listeners/MongoStartupListener.java | 97 ++
.../listeners/RestoreHandlerStartupListener.java | 50 +
.../backendapi/healthcheck/MongoHealthCheck.java | 46 +
.../backendapi/interceptor/AuditInterceptor.java | 127 +++
.../interceptor/BudgetLimitInterceptor.java | 64 ++
.../interceptor/ProjectAdminInterceptor.java | 74 ++
.../backendapi/modules/CloudProviderModule.java | 112 ++
.../epam/datalab/backendapi/modules/DevModule.java | 197 ++++
.../datalab/backendapi/modules/ModuleFactory.java | 51 +
.../backendapi/modules/ProductionModule.java | 176 +++
.../resources/ApplicationSettingResource.java | 73 ++
.../backendapi/resources/AuditResource.java | 70 ++
.../backendapi/resources/BackupResource.java | 90 ++
.../backendapi/resources/BillingResource.java | 72 ++
.../backendapi/resources/BucketResource.java | 163 +++
.../backendapi/resources/EndpointResource.java | 151 +++
.../backendapi/resources/EnvironmentResource.java | 110 ++
.../backendapi/resources/ExploratoryResource.java | 175 +++
.../backendapi/resources/GitCredsResource.java | 78 ++
.../resources/ImageExploratoryResource.java | 110 ++
.../resources/InfrastructureInfoResource.java | 95 ++
.../resources/InfrastructureTemplateResource.java | 78 ++
.../backendapi/resources/KeycloakResource.java | 140 +++
.../resources/LibExploratoryResource.java | 217 ++++
.../backendapi/resources/ProjectResource.java | 244 +++++
.../backendapi/resources/SchedulerJobResource.java | 187 ++++
.../backendapi/resources/SystemInfoResource.java | 56 +
.../backendapi/resources/UserGroupResource.java | 87 ++
.../backendapi/resources/UserRoleResource.java | 63 ++
.../backendapi/resources/UserSettingsResource.java | 84 ++
.../resources/aws/ComputationalResourceAws.java | 272 +++++
.../resources/azure/AzureOauthResource.java | 60 ++
.../azure/ComputationalResourceAzure.java | 181 ++++
.../resources/callback/BackupCallback.java | 59 +
.../callback/CheckInactivityCallback.java | 69 ++
.../resources/callback/ComputationalCallback.java | 96 ++
.../callback/EnvironmentStatusCallback.java | 71 ++
.../resources/callback/ExploratoryCallback.java | 137 +++
.../resources/callback/GitCredsCallback.java | 64 ++
.../resources/callback/ImageCallback.java | 71 ++
.../resources/callback/LibraryCallback.java | 98 ++
.../resources/callback/ProjectCallback.java | 73 ++
.../resources/callback/ReuploadKeyCallback.java | 56 +
.../backendapi/resources/dto/BackupFormDTO.java | 40 +
.../backendapi/resources/dto/BackupInfoRecord.java | 45 +
.../backendapi/resources/dto/BillingFilter.java | 58 +
.../backendapi/resources/dto/BucketDeleteDTO.java | 38 +
.../resources/dto/BucketDownloadDTO.java | 35 +
.../resources/dto/ComputationalCreateFormDTO.java | 64 ++
.../resources/dto/ComputationalTemplatesDTO.java | 35 +
.../resources/dto/ExploratoryActionFormDTO.java | 44 +
.../resources/dto/ExploratoryCreateFormDTO.java | 188 ++++
.../resources/dto/ExploratoryCreatePopUp.java | 35 +
.../dto/ExploratoryImageCreateFormDTO.java | 39 +
.../backendapi/resources/dto/FolderUploadDTO.java | 35 +
.../datalab/backendapi/resources/dto/GroupDTO.java | 37 +
.../backendapi/resources/dto/HealthStatusDTO.java | 85 ++
.../backendapi/resources/dto/HealthStatusEnum.java | 44 +
.../resources/dto/HealthStatusPageDTO.java | 59 +
.../resources/dto/HealthStatusResource.java | 110 ++
.../backendapi/resources/dto/ImageInfoRecord.java | 37 +
.../datalab/backendapi/resources/dto/KeysDTO.java | 31 +
.../backendapi/resources/dto/LibInfoRecord.java | 43 +
.../resources/dto/LibInstallFormDTO.java | 48 +
.../datalab/backendapi/resources/dto/LibKey.java | 39 +
.../resources/dto/LibraryAutoCompleteDTO.java | 33 +
.../backendapi/resources/dto/LibraryDTO.java | 32 +
.../backendapi/resources/dto/LibraryStatus.java | 47 +
.../resources/dto/ProjectActionFormDTO.java | 36 +
.../resources/dto/ProjectInfrastructureInfo.java | 51 +
.../backendapi/resources/dto/QuotaUsageDTO.java | 32 +
.../resources/dto/SearchLibsFormDTO.java | 46 +
.../dto/SparkStandaloneClusterCreateForm.java | 42 +
.../dto/SparkStandaloneConfiguration.java | 33 +
.../backendapi/resources/dto/SystemInfoDto.java | 44 +
.../backendapi/resources/dto/UpdateGroupDTO.java | 38 +
.../resources/dto/UpdateRoleGroupDto.java | 36 +
.../resources/dto/UpdateUserGroupDto.java | 36 +
.../datalab/backendapi/resources/dto/UserDTO.java | 39 +
.../backendapi/resources/dto/UserGroupDto.java | 36 +
.../backendapi/resources/dto/UserResourceInfo.java | 57 +
.../backendapi/resources/dto/UserRoleDto.java | 64 ++
.../dto/aws/AwsComputationalCreateForm.java | 57 +
.../resources/dto/aws/AwsEmrConfiguration.java | 49 +
.../dto/gcp/GcpComputationalCreateForm.java | 53 +
.../dto/gcp/GcpDataprocConfiguration.java | 42 +
.../resources/gcp/ComputationalResourceGcp.java | 259 +++++
.../backendapi/resources/gcp/GcpOauthResource.java | 65 ++
.../epam/datalab/backendapi/roles/RoleType.java | 55 +
.../epam/datalab/backendapi/roles/UserRole.java | 142 +++
.../epam/datalab/backendapi/roles/UserRoles.java | 332 ++++++
.../schedulers/CheckApplicationQuoteScheduler.java | 45 +
.../schedulers/CheckInactivityScheduledJob.java | 46 +
.../schedulers/CheckProjectQuoteScheduler.java | 52 +
.../schedulers/CheckUserQuoteScheduler.java | 49 +
.../schedulers/billing/BillingScheduler.java | 52 +
.../computational/StartComputationalJob.java | 44 +
.../computational/StopComputationalJob.java | 42 +
.../computational/TerminateComputationalJob.java | 41 +
.../endpoint/CheckEndpointStatusScheduler.java | 66 ++
.../exploratory/StartExploratoryJob.java | 45 +
.../schedulers/exploratory/StopExploratoryJob.java | 46 +
.../schedulers/internal/ManagedScheduler.java | 115 ++
.../backendapi/schedulers/internal/Scheduled.java | 39 +
.../backendapi/service/AccessKeyService.java | 28 +
.../service/ApplicationSettingService.java | 31 +
.../service/ApplicationSettingServiceImpl.java | 46 +
.../datalab/backendapi/service/AuditService.java | 34 +
.../datalab/backendapi/service/BackupService.java | 37 +
.../datalab/backendapi/service/BillingService.java | 43 +
.../datalab/backendapi/service/BucketService.java | 39 +
.../backendapi/service/ComputationalService.java | 74 ++
.../backendapi/service/EndpointService.java | 48 +
.../backendapi/service/EnvironmentService.java | 46 +
.../backendapi/service/ExploratoryService.java | 62 ++
.../backendapi/service/ExternalLibraryService.java | 28 +
.../backendapi/service/GitCredentialService.java | 29 +
.../backendapi/service/GuacamoleService.java | 30 +
.../service/ImageExploratoryService.java | 39 +
.../backendapi/service/InactivityService.java | 33 +
.../service/InfrastructureInfoService.java | 35 +
.../service/InfrastructureTemplateService.java | 32 +
.../backendapi/service/KeycloakService.java | 32 +
.../backendapi/service/KeycloakServiceImpl.java | 111 ++
.../datalab/backendapi/service/LibraryService.java | 42 +
.../datalab/backendapi/service/ProjectService.java | 61 ++
.../backendapi/service/ReuploadKeyService.java | 28 +
.../backendapi/service/SchedulerJobService.java | 92 ++
.../backendapi/service/SecurityService.java | 30 +
.../backendapi/service/SecurityServiceImpl.java | 77 ++
.../backendapi/service/SystemInfoService.java | 27 +
.../datalab/backendapi/service/TagService.java | 29 +
.../datalab/backendapi/service/TagServiceImpl.java | 41 +
.../backendapi/service/UserGroupService.java | 37 +
.../backendapi/service/UserRoleService.java | 34 +
.../backendapi/service/UserRoleServiceImpl.java | 63 ++
.../backendapi/service/UserSettingService.java | 35 +
.../backendapi/service/UserSettingServiceImpl.java | 48 +
.../service/impl/AccessKeyServiceImpl.java | 59 +
.../backendapi/service/impl/AuditServiceImpl.java | 63 ++
.../backendapi/service/impl/BackupServiceImpl.java | 69 ++
.../service/impl/BillingServiceImpl.java | 383 +++++++
.../backendapi/service/impl/BucketServiceImpl.java | 175 +++
.../service/impl/ComputationalServiceImpl.java | 396 +++++++
.../service/impl/EndpointServiceImpl.java | 193 ++++
.../service/impl/EnvironmentServiceImpl.java | 233 ++++
.../service/impl/ExploratoryServiceImpl.java | 467 ++++++++
.../service/impl/GitCredentialServiceImpl.java | 119 +++
.../service/impl/GuacamoleServiceImpl.java | 91 ++
.../service/impl/ImageExploratoryServiceImpl.java | 177 +++
.../service/impl/InactivityServiceImpl.java | 111 ++
.../impl/InfrastructureInfoServiceImpl.java | 193 ++++
.../impl/InfrastructureTemplateServiceImpl.java | 246 +++++
.../service/impl/LibraryServiceImpl.java | 311 ++++++
.../service/impl/MavenCentralLibraryService.java | 88 ++
.../service/impl/ProjectServiceImpl.java | 386 +++++++
.../service/impl/ReuploadKeyServiceImpl.java | 96 ++
.../service/impl/SchedulerJobServiceImpl.java | 514 +++++++++
.../service/impl/SystemInfoServiceImpl.java | 100 ++
.../service/impl/UserGroupServiceImpl.java | 228 ++++
.../servlet/guacamole/GuacamoleServlet.java | 97 ++
.../epam/datalab/backendapi/util/BillingUtils.java | 249 +++++
.../epam/datalab/backendapi/util/CSVFormatter.java | 67 ++
.../datalab/backendapi/util/DateRemoverUtil.java | 40 +
.../epam/datalab/backendapi/util/KeycloakUtil.java | 41 +
.../datalab/backendapi/util/RequestBuilder.java | 653 ++++++++++++
.../validation/MavenLibraryNameValidator.java | 41 +
.../validation/SchedulerJobDTOValidator.java | 47 +
.../validation/annotation/LibNameValid.java | 42 +
.../annotation/SchedulerJobDTOValid.java | 42 +
.../dlab/backendapi/SelfServiceApplication.java | 217 ----
.../com/epam/dlab/backendapi/annotation/Audit.java | 36 -
.../dlab/backendapi/annotation/BudgetLimited.java | 34 -
.../com/epam/dlab/backendapi/annotation/Info.java | 30 -
.../epam/dlab/backendapi/annotation/Project.java | 31 -
.../dlab/backendapi/annotation/ProjectAdmin.java | 30 -
.../dlab/backendapi/annotation/ResourceName.java | 30 -
.../com/epam/dlab/backendapi/annotation/User.java | 30 -
.../backendapi/auth/KeycloakAuthenticator.java | 52 -
.../auth/SelfServiceSecurityAuthorizer.java | 34 -
.../filters/DropwizardBearerTokenFilterImpl.java | 39 -
.../dlab/backendapi/conf/CloudConfiguration.java | 59 -
.../backendapi/conf/KeycloakConfiguration.java | 27 -
.../conf/SelfServiceApplicationConfiguration.java | 264 -----
.../com/epam/dlab/backendapi/dao/AuditDAO.java | 31 -
.../com/epam/dlab/backendapi/dao/AuditDAOImpl.java | 172 ---
.../com/epam/dlab/backendapi/dao/BackupDAO.java | 35 -
.../epam/dlab/backendapi/dao/BackupDAOImpl.java | 65 --
.../epam/dlab/backendapi/dao/BaseBillingDAO.java | 252 -----
.../java/com/epam/dlab/backendapi/dao/BaseDAO.java | 526 ---------
.../com/epam/dlab/backendapi/dao/BillingDAO.java | 53 -
.../epam/dlab/backendapi/dao/ComputationalDAO.java | 403 -------
.../com/epam/dlab/backendapi/dao/DockerDAO.java | 40 -
.../com/epam/dlab/backendapi/dao/EndpointDAO.java | 53 -
.../epam/dlab/backendapi/dao/EndpointDAOImpl.java | 90 --
.../java/com/epam/dlab/backendapi/dao/EnvDAO.java | 512 ---------
.../epam/dlab/backendapi/dao/ExploratoryDAO.java | 479 ---------
.../dlab/backendapi/dao/ExploratoryLibDAO.java | 402 -------
.../com/epam/dlab/backendapi/dao/GitCredsDAO.java | 90 --
.../dlab/backendapi/dao/ImageExploratoryDAO.java | 46 -
.../backendapi/dao/ImageExploratoryDAOImpl.java | 155 ---
.../com/epam/dlab/backendapi/dao/IndexCreator.java | 42 -
.../epam/dlab/backendapi/dao/MongoCollections.java | 58 -
.../com/epam/dlab/backendapi/dao/MongoSetting.java | 107 --
.../com/epam/dlab/backendapi/dao/ProjectDAO.java | 59 -
.../epam/dlab/backendapi/dao/ProjectDAOImpl.java | 190 ----
.../com/epam/dlab/backendapi/dao/RequestIdDAO.java | 67 --
.../epam/dlab/backendapi/dao/SchedulerJobDAO.java | 248 -----
.../com/epam/dlab/backendapi/dao/SecurityDAO.java | 118 --
.../com/epam/dlab/backendapi/dao/SettingsDAO.java | 391 -------
.../com/epam/dlab/backendapi/dao/UserGroupDAO.java | 33 -
.../epam/dlab/backendapi/dao/UserGroupDAOImpl.java | 68 --
.../com/epam/dlab/backendapi/dao/UserRoleDAO.java | 51 -
.../epam/dlab/backendapi/dao/UserRoleDAOImpl.java | 215 ----
.../epam/dlab/backendapi/dao/UserSettingsDAO.java | 77 --
.../dlab/backendapi/domain/AuditActionEnum.java | 24 -
.../dlab/backendapi/domain/AuditCreateDTO.java | 37 -
.../com/epam/dlab/backendapi/domain/AuditDTO.java | 39 -
.../dlab/backendapi/domain/AuditPaginationDTO.java | 43 -
.../backendapi/domain/AuditResourceTypeEnum.java | 24 -
.../dlab/backendapi/domain/AutoCompleteEnum.java | 24 -
.../epam/dlab/backendapi/domain/BillingReport.java | 45 -
.../dlab/backendapi/domain/BillingReportLine.java | 56 -
.../com/epam/dlab/backendapi/domain/BudgetDTO.java | 36 -
.../dlab/backendapi/domain/CreateProjectDTO.java | 44 -
.../epam/dlab/backendapi/domain/EndpointDTO.java | 50 -
.../backendapi/domain/EndpointResourcesDTO.java | 33 -
.../backendapi/domain/ExploratoryLibCache.java | 240 -----
.../dlab/backendapi/domain/ExploratoryLibList.java | 250 -----
.../domain/MavenSearchArtifactResponse.java | 82 --
.../dlab/backendapi/domain/NotebookTemplate.java | 40 -
.../epam/dlab/backendapi/domain/ProjectDTO.java | 104 --
.../dlab/backendapi/domain/ProjectEndpointDTO.java | 31 -
.../com/epam/dlab/backendapi/domain/RequestId.java | 84 --
.../epam/dlab/backendapi/domain/RequestIdDTO.java | 121 ---
.../domain/SchedulerConfigurationData.java | 31 -
.../backendapi/domain/UpdateProjectBudgetDTO.java | 35 -
.../dlab/backendapi/domain/UpdateProjectDTO.java | 40 -
.../dropwizard/bundles/DlabKeycloakBundle.java | 58 -
.../dropwizard/listeners/MongoStartupListener.java | 97 --
.../listeners/RestoreHandlerStartupListener.java | 50 -
.../backendapi/healthcheck/MongoHealthCheck.java | 46 -
.../backendapi/interceptor/AuditInterceptor.java | 127 ---
.../interceptor/BudgetLimitInterceptor.java | 64 --
.../interceptor/ProjectAdminInterceptor.java | 74 --
.../backendapi/modules/CloudProviderModule.java | 112 --
.../epam/dlab/backendapi/modules/DevModule.java | 197 ----
.../dlab/backendapi/modules/ModuleFactory.java | 51 -
.../dlab/backendapi/modules/ProductionModule.java | 176 ---
.../resources/ApplicationSettingResource.java | 68 --
.../dlab/backendapi/resources/AuditResource.java | 70 --
.../dlab/backendapi/resources/BackupResource.java | 85 --
.../dlab/backendapi/resources/BillingResource.java | 72 --
.../dlab/backendapi/resources/BucketResource.java | 163 ---
.../backendapi/resources/EndpointResource.java | 151 ---
.../backendapi/resources/EnvironmentResource.java | 110 --
.../backendapi/resources/ExploratoryResource.java | 174 ---
.../backendapi/resources/GitCredsResource.java | 78 --
.../resources/ImageExploratoryResource.java | 110 --
.../resources/InfrastructureInfoResource.java | 95 --
.../resources/InfrastructureTemplateResource.java | 74 --
.../backendapi/resources/KeycloakResource.java | 140 ---
.../resources/LibExploratoryResource.java | 217 ----
.../dlab/backendapi/resources/ProjectResource.java | 244 -----
.../backendapi/resources/SchedulerJobResource.java | 187 ----
.../backendapi/resources/SystemInfoResource.java | 56 -
.../backendapi/resources/UserGroupResource.java | 87 --
.../backendapi/resources/UserRoleResource.java | 63 --
.../backendapi/resources/UserSettingsResource.java | 79 --
.../resources/aws/ComputationalResourceAws.java | 272 -----
.../resources/azure/AzureOauthResource.java | 56 -
.../azure/ComputationalResourceAzure.java | 180 ----
.../resources/callback/BackupCallback.java | 59 -
.../callback/CheckInactivityCallback.java | 69 --
.../resources/callback/ComputationalCallback.java | 96 --
.../callback/EnvironmentStatusCallback.java | 71 --
.../resources/callback/ExploratoryCallback.java | 137 ---
.../resources/callback/GitCredsCallback.java | 64 --
.../resources/callback/ImageCallback.java | 71 --
.../resources/callback/LibraryCallback.java | 98 --
.../resources/callback/ProjectCallback.java | 73 --
.../resources/callback/ReuploadKeyCallback.java | 56 -
.../backendapi/resources/dto/BackupFormDTO.java | 40 -
.../backendapi/resources/dto/BackupInfoRecord.java | 45 -
.../backendapi/resources/dto/BillingFilter.java | 58 -
.../backendapi/resources/dto/BucketDeleteDTO.java | 38 -
.../resources/dto/BucketDownloadDTO.java | 35 -
.../resources/dto/ComputationalCreateFormDTO.java | 64 --
.../resources/dto/ComputationalTemplatesDTO.java | 35 -
.../resources/dto/ExploratoryActionFormDTO.java | 44 -
.../resources/dto/ExploratoryCreateFormDTO.java | 188 ----
.../resources/dto/ExploratoryCreatePopUp.java | 35 -
.../dto/ExploratoryImageCreateFormDTO.java | 39 -
.../backendapi/resources/dto/FolderUploadDTO.java | 35 -
.../dlab/backendapi/resources/dto/GroupDTO.java | 37 -
.../backendapi/resources/dto/HealthStatusDTO.java | 72 --
.../backendapi/resources/dto/HealthStatusEnum.java | 42 -
.../resources/dto/HealthStatusPageDTO.java | 59 -
.../resources/dto/HealthStatusResource.java | 91 --
.../backendapi/resources/dto/ImageInfoRecord.java | 37 -
.../dlab/backendapi/resources/dto/KeysDTO.java | 31 -
.../backendapi/resources/dto/LibInfoRecord.java | 43 -
.../resources/dto/LibInstallFormDTO.java | 48 -
.../epam/dlab/backendapi/resources/dto/LibKey.java | 39 -
.../resources/dto/LibraryAutoCompleteDTO.java | 33 -
.../dlab/backendapi/resources/dto/LibraryDTO.java | 32 -
.../backendapi/resources/dto/LibraryStatus.java | 47 -
.../resources/dto/ProjectActionFormDTO.java | 36 -
.../resources/dto/ProjectInfrastructureInfo.java | 51 -
.../backendapi/resources/dto/QuotaUsageDTO.java | 32 -
.../resources/dto/SearchLibsFormDTO.java | 46 -
.../dto/SparkStandaloneClusterCreateForm.java | 42 -
.../dto/SparkStandaloneConfiguration.java | 33 -
.../backendapi/resources/dto/SystemInfoDto.java | 44 -
.../backendapi/resources/dto/UpdateGroupDTO.java | 38 -
.../resources/dto/UpdateRoleGroupDto.java | 36 -
.../resources/dto/UpdateUserGroupDto.java | 36 -
.../dlab/backendapi/resources/dto/UserDTO.java | 39 -
.../backendapi/resources/dto/UserGroupDto.java | 36 -
.../backendapi/resources/dto/UserResourceInfo.java | 57 -
.../dlab/backendapi/resources/dto/UserRoleDto.java | 64 --
.../dto/aws/AwsComputationalCreateForm.java | 57 -
.../resources/dto/aws/AwsEmrConfiguration.java | 49 -
.../dto/gcp/GcpComputationalCreateForm.java | 53 -
.../dto/gcp/GcpDataprocConfiguration.java | 42 -
.../resources/gcp/ComputationalResourceGcp.java | 259 -----
.../backendapi/resources/gcp/GcpOauthResource.java | 61 --
.../com/epam/dlab/backendapi/roles/RoleType.java | 55 -
.../com/epam/dlab/backendapi/roles/UserRole.java | 142 ---
.../com/epam/dlab/backendapi/roles/UserRoles.java | 332 ------
.../schedulers/CheckApplicationQuoteScheduler.java | 45 -
.../schedulers/CheckInactivityScheduledJob.java | 46 -
.../schedulers/CheckProjectQuoteScheduler.java | 52 -
.../schedulers/CheckUserQuoteScheduler.java | 49 -
.../schedulers/billing/BillingScheduler.java | 52 -
.../computational/StartComputationalJob.java | 44 -
.../computational/StopComputationalJob.java | 42 -
.../computational/TerminateComputationalJob.java | 41 -
.../endpoint/CheckEndpointStatusScheduler.java | 66 --
.../exploratory/StartExploratoryJob.java | 45 -
.../schedulers/exploratory/StopExploratoryJob.java | 46 -
.../schedulers/internal/ManagedScheduler.java | 109 --
.../backendapi/schedulers/internal/Scheduled.java | 39 -
.../dlab/backendapi/service/AccessKeyService.java | 28 -
.../service/ApplicationSettingService.java | 31 -
.../service/ApplicationSettingServiceImpl.java | 45 -
.../epam/dlab/backendapi/service/AuditService.java | 34 -
.../dlab/backendapi/service/BackupService.java | 37 -
.../dlab/backendapi/service/BillingService.java | 43 -
.../dlab/backendapi/service/BucketService.java | 39 -
.../backendapi/service/ComputationalService.java | 74 --
.../dlab/backendapi/service/EndpointService.java | 48 -
.../backendapi/service/EnvironmentService.java | 46 -
.../backendapi/service/ExploratoryService.java | 62 --
.../backendapi/service/ExternalLibraryService.java | 28 -
.../backendapi/service/GitCredentialService.java | 29 -
.../dlab/backendapi/service/GuacamoleService.java | 30 -
.../service/ImageExploratoryService.java | 39 -
.../dlab/backendapi/service/InactivityService.java | 33 -
.../service/InfrastructureInfoService.java | 35 -
.../service/InfrastructureTemplateService.java | 32 -
.../dlab/backendapi/service/KeycloakService.java | 32 -
.../backendapi/service/KeycloakServiceImpl.java | 111 --
.../dlab/backendapi/service/LibraryService.java | 42 -
.../dlab/backendapi/service/ProjectService.java | 61 --
.../backendapi/service/ReuploadKeyService.java | 28 -
.../backendapi/service/SchedulerJobService.java | 92 --
.../dlab/backendapi/service/SecurityService.java | 30 -
.../backendapi/service/SecurityServiceImpl.java | 77 --
.../dlab/backendapi/service/SystemInfoService.java | 27 -
.../epam/dlab/backendapi/service/TagService.java | 29 -
.../dlab/backendapi/service/TagServiceImpl.java | 41 -
.../dlab/backendapi/service/UserGroupService.java | 37 -
.../dlab/backendapi/service/UserRoleService.java | 34 -
.../backendapi/service/UserRoleServiceImpl.java | 63 --
.../backendapi/service/UserSettingService.java | 35 -
.../backendapi/service/UserSettingServiceImpl.java | 48 -
.../service/impl/AccessKeyServiceImpl.java | 59 -
.../backendapi/service/impl/AuditServiceImpl.java | 63 --
.../backendapi/service/impl/BackupServiceImpl.java | 69 --
.../service/impl/BillingServiceImpl.java | 383 -------
.../backendapi/service/impl/BucketServiceImpl.java | 175 ---
.../service/impl/ComputationalServiceImpl.java | 396 -------
.../service/impl/EndpointServiceImpl.java | 193 ----
.../service/impl/EnvironmentServiceImpl.java | 233 ----
.../service/impl/ExploratoryServiceImpl.java | 467 --------
.../service/impl/GitCredentialServiceImpl.java | 119 ---
.../service/impl/GuacamoleServiceImpl.java | 91 --
.../service/impl/ImageExploratoryServiceImpl.java | 177 ---
.../service/impl/InactivityServiceImpl.java | 111 --
.../impl/InfrastructureInfoServiceImpl.java | 193 ----
.../impl/InfrastructureTemplateServiceImpl.java | 246 -----
.../service/impl/LibraryServiceImpl.java | 311 ------
.../service/impl/MavenCentralLibraryService.java | 88 --
.../service/impl/ProjectServiceImpl.java | 386 -------
.../service/impl/ReuploadKeyServiceImpl.java | 96 --
.../service/impl/SchedulerJobServiceImpl.java | 514 ---------
.../service/impl/SystemInfoServiceImpl.java | 100 --
.../service/impl/UserGroupServiceImpl.java | 228 ----
.../servlet/guacamole/GuacamoleServlet.java | 97 --
.../epam/dlab/backendapi/util/BillingUtils.java | 250 -----
.../epam/dlab/backendapi/util/CSVFormatter.java | 67 --
.../epam/dlab/backendapi/util/DateRemoverUtil.java | 40 -
.../epam/dlab/backendapi/util/KeycloakUtil.java | 41 -
.../epam/dlab/backendapi/util/RequestBuilder.java | 653 ------------
.../validation/MavenLibraryNameValidator.java | 41 -
.../validation/SchedulerJobDTOValidator.java | 47 -
.../validation/annotation/LibNameValid.java | 42 -
.../annotation/SchedulerJobDTOValid.java | 42 -
.../src/main/resources/mongo/aws/mongo_roles.json | 18 +-
.../main/resources/mongo/azure/mongo_roles.json | 12 +-
.../src/main/resources/mongo/gcp/mongo_roles.json | 20 +-
.../src/main/resources/quartz.properties | 2 +-
.../com/epam/{dlab => datalab}/backendapi/.gitkeep | 0
.../backendapi/domain/ExploratoryLibListTest.java | 67 ++
.../resources/ApplicationSettingResourceTest.java | 105 ++
.../backendapi/resources/AuditResourceTest.java | 88 ++
.../backendapi/resources/BackupResourceTest.java | 228 ++++
.../resources/EnvironmentResourceTest.java | 291 +++++
.../resources/ExploratoryResourceTest.java | 325 ++++++
.../backendapi/resources/GitCredsResourceTest.java | 180 ++++
.../resources/ImageExploratoryResourceTest.java | 282 +++++
.../resources/InfrastructureInfoResourceTest.java | 205 ++++
.../InfrastructureTemplateResourceTest.java | 182 ++++
.../backendapi/resources/KeycloakResourceTest.java | 70 ++
.../resources/LibExploratoryResourceTest.java | 443 ++++++++
.../backendapi/resources/ProjectResourceTest.java | 292 +++++
.../resources/SchedulerJobResourceTest.java | 341 ++++++
.../resources/SystemInfoResourceTest.java | 101 ++
.../datalab/backendapi/resources/TestBase.java | 92 ++
.../resources/UserGroupResourceTest.java | 186 ++++
.../backendapi/resources/UserRoleResourceTest.java | 105 ++
.../resources/UserSettingsResourceTest.java | 166 +++
.../datalab/backendapi/roles/UserRolesTest.java | 88 ++
.../CheckApplicationQuoteSchedulerTest.java | 69 ++
.../schedulers/CheckUserQuoteSchedulerTest.java | 81 ++
.../service/ApplicationSettingServiceImplTest.java | 74 ++
.../service/KeycloakServiceImplTest.java | 108 ++
.../backendapi/service/ProjectServiceImplTest.java | 426 ++++++++
.../service/SecurityServiceImplTest.java | 134 +++
.../backendapi/service/TagServiceImplTest.java | 70 ++
.../service/UserRoleServiceImplTest.java | 91 ++
.../service/UserSettingServiceImplTest.java | 89 ++
.../service/impl/AccessKeyServiceImplTest.java | 66 ++
.../service/impl/AuditServiceImplTest.java | 106 ++
.../service/impl/BackupServiceImplTest.java | 151 +++
.../service/impl/BillingServiceImplTest.java | 739 +++++++++++++
.../service/impl/BucketServiceImplTest.java | 262 +++++
.../service/impl/ComputationalServiceImplTest.java | 784 ++++++++++++++
.../service/impl/EndpointServiceImplTest.java | 310 ++++++
.../service/impl/EnvironmentServiceImplTest.java | 369 +++++++
.../service/impl/ExploratoryServiceImplTest.java | 549 ++++++++++
.../service/impl/GitCredentialServiceImplTest.java | 182 ++++
.../impl/ImageExploratoryServiceImplTest.java | 352 ++++++
.../impl/InfrastructureInfoServiceImplTest.java | 387 +++++++
.../InfrastructureTemplateServiceBaseTest.java | 226 ++++
.../service/impl/LibraryServiceImplTest.java | 490 +++++++++
.../impl/MavenCentralLibraryServiceTest.java | 92 ++
.../service/impl/ReuploadKeyServiceImplTest.java | 183 ++++
.../service/impl/SchedulerJobServiceImplTest.java | 1125 ++++++++++++++++++++
.../service/impl/SystemInfoServiceImplTest.java | 99 ++
.../service/impl/UserGroupServiceImplTest.java | 182 ++++
.../datalab/backendapi/util/CSVFormatterTest.java | 46 +
.../backendapi/util/DateRemoverUtilTest.java | 44 +
.../backendapi/domain/ExploratoryLibListTest.java | 67 --
.../resources/ApplicationSettingResourceTest.java | 102 --
.../backendapi/resources/AuditResourceTest.java | 88 --
.../backendapi/resources/BackupResourceTest.java | 223 ----
.../resources/EnvironmentResourceTest.java | 291 -----
.../resources/ExploratoryResourceTest.java | 325 ------
.../backendapi/resources/GitCredsResourceTest.java | 174 ---
.../resources/ImageExploratoryResourceTest.java | 281 -----
.../resources/InfrastructureInfoResourceTest.java | 205 ----
.../InfrastructureTemplateResourceTest.java | 177 ---
.../backendapi/resources/KeycloakResourceTest.java | 70 --
.../resources/LibExploratoryResourceTest.java | 443 --------
.../backendapi/resources/ProjectResourceTest.java | 292 -----
.../resources/SchedulerJobResourceTest.java | 341 ------
.../resources/SystemInfoResourceTest.java | 97 --
.../epam/dlab/backendapi/resources/TestBase.java | 92 --
.../resources/UserGroupResourceTest.java | 186 ----
.../backendapi/resources/UserRoleResourceTest.java | 105 --
.../resources/UserSettingsResourceTest.java | 159 ---
.../epam/dlab/backendapi/roles/UserRolesTest.java | 88 --
.../CheckApplicationQuoteSchedulerTest.java | 66 --
.../schedulers/CheckUserQuoteSchedulerTest.java | 77 --
.../service/ApplicationSettingServiceImplTest.java | 72 --
.../service/KeycloakServiceImplTest.java | 108 --
.../backendapi/service/ProjectServiceImplTest.java | 426 --------
.../service/SecurityServiceImplTest.java | 134 ---
.../backendapi/service/TagServiceImplTest.java | 70 --
.../service/UserRoleServiceImplTest.java | 91 --
.../service/UserSettingServiceImplTest.java | 86 --
.../service/impl/AccessKeyServiceImplTest.java | 66 --
.../service/impl/AuditServiceImplTest.java | 106 --
.../service/impl/BackupServiceImplTest.java | 151 ---
.../service/impl/BillingServiceImplTest.java | 739 -------------
.../service/impl/BucketServiceImplTest.java | 262 -----
.../service/impl/ComputationalServiceImplTest.java | 784 --------------
.../service/impl/EndpointServiceImplTest.java | 310 ------
.../service/impl/EnvironmentServiceImplTest.java | 369 -------
.../service/impl/ExploratoryServiceImplTest.java | 549 ----------
.../service/impl/GitCredentialServiceImplTest.java | 175 ---
.../impl/ImageExploratoryServiceImplTest.java | 352 ------
.../impl/InfrastructureInfoServiceImplTest.java | 387 -------
.../InfrastructureTemplateServiceBaseTest.java | 226 ----
.../service/impl/LibraryServiceImplTest.java | 490 ---------
.../impl/MavenCentralLibraryServiceTest.java | 90 --
.../service/impl/ReuploadKeyServiceImplTest.java | 183 ----
.../service/impl/SchedulerJobServiceImplTest.java | 1125 --------------------
.../service/impl/SystemInfoServiceImplTest.java | 96 --
.../service/impl/UserGroupServiceImplTest.java | 182 ----
.../dlab/backendapi/util/CSVFormatterTest.java | 46 -
.../dlab/backendapi/util/DateRemoverUtilTest.java | 44 -
1634 files changed, 71637 insertions(+), 70973 deletions(-)
diff --git a/build.properties b/build.properties
index 989bad1..275dfaa 100644
--- a/build.properties
+++ b/build.properties
@@ -16,4 +16,4 @@
# specific language governing permissions and limitations
# under the License.
#
-dlab.version=2.4
\ No newline at end of file
+datalab.version=2.4
\ No newline at end of file
diff --git a/integration-tests-cucumber/pom.xml b/integration-tests-cucumber/pom.xml
index d862bcb..4d00132 100644
--- a/integration-tests-cucumber/pom.xml
+++ b/integration-tests-cucumber/pom.xml
@@ -23,7 +23,7 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.epam.dlab</groupId>
+ <groupId>com.epam.datalab</groupId>
<artifactId>integration-tests</artifactId>
<version>1.0.0-SNAPSHOT</version>
<packaging>jar</packaging>
diff --git a/integration-tests-cucumber/src/main/java/org/apache/datalab/dto/EndpointDTO.java b/integration-tests-cucumber/src/main/java/org/apache/datalab/dto/EndpointDTO.java
new file mode 100644
index 0000000..cddbbe0
--- /dev/null
+++ b/integration-tests-cucumber/src/main/java/org/apache/datalab/dto/EndpointDTO.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datalab.dto;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+@AllArgsConstructor
+@NoArgsConstructor
+public class EndpointDTO {
+ private String name;
+ private String url;
+ private String account;
+ @JsonProperty("endpoint_tag")
+ private String tag;
+}
diff --git a/integration-tests-cucumber/src/main/java/org/apache/datalab/mongo/MongoDBHelper.java b/integration-tests-cucumber/src/main/java/org/apache/datalab/mongo/MongoDBHelper.java
new file mode 100644
index 0000000..fd2eb97
--- /dev/null
+++ b/integration-tests-cucumber/src/main/java/org/apache/datalab/mongo/MongoDBHelper.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datalab.mongo;
+
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import org.apache.datalab.util.PropertyHelper;
+
+public class MongoDBHelper {
+ private static final MongoClient client = MongoClients
+ .create(PropertyHelper.read("mongo.connection.string"));
+
+ public static void cleanCollection(String collection) {
+ client.getDatabase(PropertyHelper.read("mongo.db.name")).getCollection(collection).drop();
+ }
+}
diff --git a/integration-tests-cucumber/src/main/java/org/apache/datalab/util/JacksonMapper.java b/integration-tests-cucumber/src/main/java/org/apache/datalab/util/JacksonMapper.java
new file mode 100644
index 0000000..b2d9498
--- /dev/null
+++ b/integration-tests-cucumber/src/main/java/org/apache/datalab/util/JacksonMapper.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datalab.util;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public final class JacksonMapper {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ public static <T> String marshall(T obj) {
+ try {
+ return MAPPER.writeValueAsString(obj);
+ } catch (JsonProcessingException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+}
diff --git a/integration-tests-cucumber/src/main/java/org/apache/datalab/util/PropertyHelper.java b/integration-tests-cucumber/src/main/java/org/apache/datalab/util/PropertyHelper.java
new file mode 100644
index 0000000..b42cd61
--- /dev/null
+++ b/integration-tests-cucumber/src/main/java/org/apache/datalab/util/PropertyHelper.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datalab.util;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Properties;
+
+public class PropertyHelper {
+
+ private final static Properties PROPERTIES;
+
+ static {
+ PROPERTIES = new Properties();
+ try (InputStream inputStream = new FileInputStream(System.getProperty("config.file"))) {
+ PROPERTIES.load(inputStream);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ public static String read(String prop) {
+ return PROPERTIES.getProperty(prop);
+ }
+}
diff --git a/integration-tests-cucumber/src/main/java/org/apache/dlab/dto/EndpointDTO.java b/integration-tests-cucumber/src/main/java/org/apache/dlab/dto/EndpointDTO.java
deleted file mode 100644
index 7cfdad2..0000000
--- a/integration-tests-cucumber/src/main/java/org/apache/dlab/dto/EndpointDTO.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.dlab.dto;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@JsonIgnoreProperties(ignoreUnknown = true)
-@AllArgsConstructor
-@NoArgsConstructor
-public class EndpointDTO {
- private String name;
- private String url;
- private String account;
- @JsonProperty("endpoint_tag")
- private String tag;
-}
diff --git a/integration-tests-cucumber/src/main/java/org/apache/dlab/mongo/MongoDBHelper.java b/integration-tests-cucumber/src/main/java/org/apache/dlab/mongo/MongoDBHelper.java
deleted file mode 100644
index 4903fd4..0000000
--- a/integration-tests-cucumber/src/main/java/org/apache/dlab/mongo/MongoDBHelper.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.dlab.mongo;
-
-import com.mongodb.client.MongoClient;
-import com.mongodb.client.MongoClients;
-import org.apache.dlab.util.PropertyHelper;
-
-public class MongoDBHelper {
- private static final MongoClient client = MongoClients
- .create(PropertyHelper.read("mongo.connection.string"));
-
- public static void cleanCollection(String collection) {
- client.getDatabase(PropertyHelper.read("mongo.db.name")).getCollection(collection).drop();
- }
-}
diff --git a/integration-tests-cucumber/src/main/java/org/apache/dlab/util/JacksonMapper.java b/integration-tests-cucumber/src/main/java/org/apache/dlab/util/JacksonMapper.java
deleted file mode 100644
index ae4d5ce..0000000
--- a/integration-tests-cucumber/src/main/java/org/apache/dlab/util/JacksonMapper.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.dlab.util;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-public final class JacksonMapper {
- private static final ObjectMapper MAPPER = new ObjectMapper();
-
- public static <T> String marshall(T obj) {
- try {
- return MAPPER.writeValueAsString(obj);
- } catch (JsonProcessingException e) {
- throw new IllegalArgumentException(e);
- }
- }
-}
diff --git a/integration-tests-cucumber/src/main/java/org/apache/dlab/util/PropertyHelper.java b/integration-tests-cucumber/src/main/java/org/apache/dlab/util/PropertyHelper.java
deleted file mode 100644
index 71688e2..0000000
--- a/integration-tests-cucumber/src/main/java/org/apache/dlab/util/PropertyHelper.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.dlab.util;
-
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.util.Properties;
-
-public class PropertyHelper {
-
- private final static Properties PROPERTIES;
-
- static {
- PROPERTIES = new Properties();
- try (InputStream inputStream = new FileInputStream(System.getProperty("config.file"))) {
- PROPERTIES.load(inputStream);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- public static String read(String prop) {
- return PROPERTIES.getProperty(prop);
- }
-}
diff --git a/integration-tests-cucumber/src/test/java/datalab/Constants.java b/integration-tests-cucumber/src/test/java/datalab/Constants.java
new file mode 100644
index 0000000..375d614
--- /dev/null
+++ b/integration-tests-cucumber/src/test/java/datalab/Constants.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datalab;
+
+public interface Constants {
+ String API_URI = "https://localhost:8443/api/";
+}
diff --git a/integration-tests-cucumber/src/test/java/datalab/RunCucumberTest.java b/integration-tests-cucumber/src/test/java/datalab/RunCucumberTest.java
new file mode 100644
index 0000000..4fbc70c
--- /dev/null
+++ b/integration-tests-cucumber/src/test/java/datalab/RunCucumberTest.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datalab;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(plugin = {"json:target/cucumber.json"})
+public class RunCucumberTest {
+}
diff --git a/integration-tests-cucumber/src/test/java/datalab/endpoint/EndpointSteps.java b/integration-tests-cucumber/src/test/java/datalab/endpoint/EndpointSteps.java
new file mode 100644
index 0000000..fa46141
--- /dev/null
+++ b/integration-tests-cucumber/src/test/java/datalab/endpoint/EndpointSteps.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datalab.endpoint;
+
+import com.jayway.restassured.http.ContentType;
+import com.jayway.restassured.response.Response;
+import com.jayway.restassured.specification.RequestSpecification;
+import cucumber.api.java.en.And;
+import cucumber.api.java.en.Given;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+import org.apache.datalab.dto.EndpointDTO;
+import org.apache.datalab.mongo.MongoDBHelper;
+import org.apache.datalab.util.JacksonMapper;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static com.jayway.restassured.RestAssured.given;
+import static datalab.Constants.API_URI;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.IsEqual.equalTo;
+
+public class EndpointSteps {
+ private RequestSpecification request;
+ private Response response;
+ private String name;
+
+ @Given("User try to create new endpoint with name {string} and uri {string} and account {string} and {string}")
+ public void userTryToCreateNewEndpoint(String name, String uri, String account, String tag) {
+ this.name = name;
+ request = given().body(JacksonMapper.marshall(new EndpointDTO(name, uri, account, tag)))
+ .auth()
+ .oauth2("token123")
+ .contentType(ContentType.JSON);
+
+ }
+
+ @When("User send create new endpoint request")
+ public void userSendCreateNewEndpoint() {
+ response = request.post(API_URI + "endpoint");
+ }
+
+ @Given("There is no endpoints in DataLab")
+ public void thereIsNoEndpointsInDataLab() {
+ MongoDBHelper.cleanCollection("endpoints");
+
+ }
+
+ @Then("Response status code is {int}")
+ public void responseStatusCodeIs(int code) {
+ assertThat(response.getStatusCode(), equalTo(code));
+ }
+
+ @And("Endpoint URI is present in location header")
+ public void endpointURIIsPresentInLocationHeader() {
+ assertThat(response.getHeader("Location"), equalTo(API_URI + "endpoint/" + name));
+ }
+
+ @When("User try to get information about endpoint with name {string}")
+ public void userTryToGetInformationAboutEndpointWithName(String endpoint) throws URISyntaxException {
+ response = authenticatedRequest()
+ .get(new URI(API_URI + "endpoint/" + endpoint));
+
+ }
+
+ @And("Endpoint information is successfully returned with " +
+ "name {string}, uri {string}, account {string}, and tag {string}")
+ public void endpointInformationIsSuccessfullyReturnedWithNameUriAccountAndTag(String name, String uri,
+ String account, String tag) {
+ final EndpointDTO dto = response.getBody().as(EndpointDTO.class);
+ assertThat(dto.getAccount(), equalTo(account));
+ assertThat(dto.getName(), equalTo(name));
+ assertThat(dto.getUrl(), equalTo(uri));
+ assertThat(dto.getTag(), equalTo(tag));
+
+ }
+
+ @When("User try to get information about endpoints")
+ public void userTryToGetInformationAboutEndpoints() throws URISyntaxException {
+ response = authenticatedRequest()
+ .get(new URI(API_URI + "endpoint"));
+
+ }
+
+ @And("There are endpoints with name test1 and test2")
+ public void thereAreEndpointsWithNameTestAndTest() {
+ final EndpointDTO[] endpoints = response.getBody().as(EndpointDTO[].class);
+ assertThat(2, equalTo(endpoints.length));
+ assertThat("test1", equalTo(endpoints[0].getName()));
+ assertThat("test2", equalTo(endpoints[1].getName()));
+ }
+
+ private RequestSpecification authenticatedRequest() {
+ return given()
+ .auth()
+ .oauth2("token123");
+ }
+}
diff --git a/integration-tests-cucumber/src/test/java/datalab/login/LoginSteps.java b/integration-tests-cucumber/src/test/java/datalab/login/LoginSteps.java
new file mode 100644
index 0000000..9351b33
--- /dev/null
+++ b/integration-tests-cucumber/src/test/java/datalab/login/LoginSteps.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package datalab.login;
+
+import com.jayway.restassured.http.ContentType;
+import com.jayway.restassured.response.Response;
+import com.jayway.restassured.specification.RequestSpecification;
+import cucumber.api.java.en.Given;
+import cucumber.api.java.en.Then;
+import cucumber.api.java.en.When;
+import gherkin.deps.com.google.gson.JsonObject;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static com.jayway.restassured.RestAssured.given;
+import static datalab.Constants.API_URI;
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+
+public class LoginSteps {
+
+
+ private static final String LOGIN_RESOURCE_PATH = API_URI + "user/login";
+ private RequestSpecification request;
+ private Response response;
+
+ @Given("User try to login to Datalab with {string} and {string}")
+ public void userProvidedLoginAndPassword(String username, String password) {
+ JsonObject jsonObject = new JsonObject();
+ jsonObject.addProperty("username", username);
+ jsonObject.addProperty("password", password);
+ request = given().body(jsonObject.toString()).contentType(ContentType.JSON);
+ }
+
+ @When("user try to login")
+ public void userTryToLogin() throws URISyntaxException {
+ response = request.post(new URI(LOGIN_RESOURCE_PATH));
+ }
+
+ @Then("response code is {string}")
+ public void responseCodeIs(String status) {
+ assertThat(response.getStatusCode(), equalTo(Integer.valueOf(status)));
+
+ }
+}
diff --git a/integration-tests-cucumber/src/test/java/dlab/Constants.java b/integration-tests-cucumber/src/test/java/dlab/Constants.java
deleted file mode 100644
index 8e1b6b9..0000000
--- a/integration-tests-cucumber/src/test/java/dlab/Constants.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package dlab;
-
-public interface Constants {
- String API_URI = "https://localhost:8443/api/";
-}
diff --git a/integration-tests-cucumber/src/test/java/dlab/RunCucumberTest.java b/integration-tests-cucumber/src/test/java/dlab/RunCucumberTest.java
deleted file mode 100644
index fb03b55..0000000
--- a/integration-tests-cucumber/src/test/java/dlab/RunCucumberTest.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package dlab;
-
-import cucumber.api.CucumberOptions;
-import cucumber.api.junit.Cucumber;
-import org.junit.runner.RunWith;
-
-@RunWith(Cucumber.class)
-@CucumberOptions(plugin = {"json:target/cucumber.json"})
-public class RunCucumberTest {
-}
diff --git a/integration-tests-cucumber/src/test/java/dlab/endpoint/EndpointSteps.java b/integration-tests-cucumber/src/test/java/dlab/endpoint/EndpointSteps.java
deleted file mode 100644
index 1c1d43a..0000000
--- a/integration-tests-cucumber/src/test/java/dlab/endpoint/EndpointSteps.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package dlab.endpoint;
-
-import com.jayway.restassured.http.ContentType;
-import com.jayway.restassured.response.Response;
-import com.jayway.restassured.specification.RequestSpecification;
-import cucumber.api.java.en.And;
-import cucumber.api.java.en.Given;
-import cucumber.api.java.en.Then;
-import cucumber.api.java.en.When;
-import org.apache.dlab.dto.EndpointDTO;
-import org.apache.dlab.mongo.MongoDBHelper;
-import org.apache.dlab.util.JacksonMapper;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import static com.jayway.restassured.RestAssured.given;
-import static dlab.Constants.API_URI;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.core.IsEqual.equalTo;
-
-public class EndpointSteps {
- private RequestSpecification request;
- private Response response;
- private String name;
-
- @Given("User try to create new endpoint with name {string} and uri {string} and account {string} and {string}")
- public void userTryToCreateNewEndpoint(String name, String uri, String account, String tag) {
- this.name = name;
- request = given().body(JacksonMapper.marshall(new EndpointDTO(name, uri, account, tag)))
- .auth()
- .oauth2("token123")
- .contentType(ContentType.JSON);
-
- }
-
- @When("User send create new endpoint request")
- public void userSendCreateNewEndpoint() {
- response = request.post(API_URI + "endpoint");
- }
-
- @Given("There is no endpoints in DLab")
- public void thereIsNoEndpointsInDLab() {
- MongoDBHelper.cleanCollection("endpoints");
-
- }
-
- @Then("Response status code is {int}")
- public void responseStatusCodeIs(int code) {
- assertThat(response.getStatusCode(), equalTo(code));
- }
-
- @And("Endpoint URI is present in location header")
- public void endpointURIIsPresentInLocationHeader() {
- assertThat(response.getHeader("Location"), equalTo(API_URI + "endpoint/" + name));
- }
-
- @When("User try to get information about endpoint with name {string}")
- public void userTryToGetInformationAboutEndpointWithName(String endpoint) throws URISyntaxException {
- response = authenticatedRequest()
- .get(new URI(API_URI + "endpoint/" + endpoint));
-
- }
-
- @And("Endpoint information is successfully returned with " +
- "name {string}, uri {string}, account {string}, and tag {string}")
- public void endpointInformationIsSuccessfullyReturnedWithNameUriAccountAndTag(String name, String uri,
- String account, String tag) {
- final EndpointDTO dto = response.getBody().as(EndpointDTO.class);
- assertThat(dto.getAccount(), equalTo(account));
- assertThat(dto.getName(), equalTo(name));
- assertThat(dto.getUrl(), equalTo(uri));
- assertThat(dto.getTag(), equalTo(tag));
-
- }
-
- @When("User try to get information about endpoints")
- public void userTryToGetInformationAboutEndpoints() throws URISyntaxException {
- response = authenticatedRequest()
- .get(new URI(API_URI + "endpoint"));
-
- }
-
- @And("There are endpoints with name test1 and test2")
- public void thereAreEndpointsWithNameTestAndTest() {
- final EndpointDTO[] endpoints = response.getBody().as(EndpointDTO[].class);
- assertThat(2, equalTo(endpoints.length));
- assertThat("test1", equalTo(endpoints[0].getName()));
- assertThat("test2", equalTo(endpoints[1].getName()));
- }
-
- private RequestSpecification authenticatedRequest() {
- return given()
- .auth()
- .oauth2("token123");
- }
-}
diff --git a/integration-tests-cucumber/src/test/java/dlab/login/LoginSteps.java b/integration-tests-cucumber/src/test/java/dlab/login/LoginSteps.java
deleted file mode 100644
index fd533e0..0000000
--- a/integration-tests-cucumber/src/test/java/dlab/login/LoginSteps.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package dlab.login;
-
-import com.jayway.restassured.http.ContentType;
-import com.jayway.restassured.response.Response;
-import com.jayway.restassured.specification.RequestSpecification;
-import cucumber.api.java.en.Given;
-import cucumber.api.java.en.Then;
-import cucumber.api.java.en.When;
-import gherkin.deps.com.google.gson.JsonObject;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import static com.jayway.restassured.RestAssured.given;
-import static dlab.Constants.API_URI;
-import static org.hamcrest.core.IsEqual.equalTo;
-import static org.junit.Assert.assertThat;
-
-public class LoginSteps {
-
-
- private static final String LOGIN_RESOURCE_PATH = API_URI + "user/login";
- private RequestSpecification request;
- private Response response;
-
- @Given("User try to login to Dlab with {string} and {string}")
- public void userProvidedLoginAndPassword(String username, String password) {
- JsonObject jsonObject = new JsonObject();
- jsonObject.addProperty("username", username);
- jsonObject.addProperty("password", password);
- request = given().body(jsonObject.toString()).contentType(ContentType.JSON);
- }
-
- @When("user try to login")
- public void userTryToLogin() throws URISyntaxException {
- response = request.post(new URI(LOGIN_RESOURCE_PATH));
- }
-
- @Then("response code is {string}")
- public void responseCodeIs(String status) {
- assertThat(response.getStatusCode(), equalTo(Integer.valueOf(status)));
-
- }
-}
diff --git a/integration-tests-cucumber/src/test/resources/config.properties b/integration-tests-cucumber/src/test/resources/config.properties
index d0cfc24..e1a45b9 100644
--- a/integration-tests-cucumber/src/test/resources/config.properties
+++ b/integration-tests-cucumber/src/test/resources/config.properties
@@ -16,5 +16,5 @@
# specific language governing permissions and limitations
# under the License.
#
-mongo.connection.string=mongodb://localhost:27017/DLAB
-mongo.db.name=DLAB
\ No newline at end of file
+mongo.connection.string=mongodb://localhost:27017/DATALAB
+mongo.db.name=DATALAB
\ No newline at end of file
diff --git a/integration-tests-cucumber/src/test/resources/datalab/endpoint.feature b/integration-tests-cucumber/src/test/resources/datalab/endpoint.feature
new file mode 100644
index 0000000..0cca194
--- /dev/null
+++ b/integration-tests-cucumber/src/test/resources/datalab/endpoint.feature
@@ -0,0 +1,75 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+Feature: Endpoint management in DataLab
+ Such feature allowed to manage endpoint inside DataLab
+
+ Scenario Outline: Create new endpoint when it does not exist
+
+ Given There is no endpoints in DataLab
+ And User try to create new endpoint with name "<name>" and uri "<uri>" and account "<account>" and "<tag>"
+ When User send create new endpoint request
+ Then Response status code is 200
+ And Endpoint URI is present in location header
+ Examples:
+ | name | uri | account | tag |
+ | test_endpoint | someuri | 123231312 | some_tag |
+
+
+ Scenario Outline: Create new endpoint when it exist already
+
+ Given There is no endpoints in DataLab
+ And User try to create new endpoint with name "<name>" and uri "<uri>" and account "<account>" and "<tag>"
+ And User send create new endpoint request
+ When User try to create new endpoint with name "<name>" and uri "<uri>" and account "<account>" and "<tag>"
+ And User send create new endpoint request
+ Then Response status code is 409
+ Examples:
+ | name | uri | account | tag |
+ | test_endpoint | someuri | 123231312 | some_tag |
+
+
+ Scenario Outline: Get information for endpoint
+
+ Given There is no endpoints in DataLab
+ And User try to create new endpoint with name "<name>" and uri "<uri>" and account "<account>" and "<tag>"
+ And User send create new endpoint request
+ When User try to get information about endpoint with name "<name>"
+ Then Response status code is 200
+ And Endpoint information is successfully returned with name "<name>", uri "<uri>", account "<account>", and tag "<tag>"
+ Examples:
+ | name | uri | account | tag |
+ | test_endpoint | someuri | 123231312 | some_tag |
+
+
+ Scenario: Get list of endpoints
+
+ Given There is no endpoints in DataLab
+ And User try to create new endpoint with name "test1" and uri "someuri1" and account "123" and "customTag1"
+ And User send create new endpoint request
+ And User try to create new endpoint with name "test2" and uri "someuri2" and account "1233" and "customTag4"
+ And User send create new endpoint request
+ When User try to get information about endpoints
+ Then Response status code is 200
+ And There are endpoints with name test1 and test2
+
+ Scenario: Get not endpoint that does not exist
+
+ Given There is no endpoints in DataLab
+ When User try to get information about endpoint with name "someName"
+ Then Response status code is 404
diff --git a/integration-tests-cucumber/src/test/resources/datalab/login.feature b/integration-tests-cucumber/src/test/resources/datalab/login.feature
new file mode 100644
index 0000000..243a6f3
--- /dev/null
+++ b/integration-tests-cucumber/src/test/resources/datalab/login.feature
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+Feature: DataLab login API
+ Used to check DataLab login flow
+
+ Scenario Outline: User try to login to DataLab
+ Given User try to login to Datalab with "<username>" and "<password>"
+ When user try to login
+ Then response code is "<status>"
+
+ Examples:
+ | username | password | status |
+ | test | pass | 200 |
+ | not_valid_user | pass | 401 |
\ No newline at end of file
diff --git a/integration-tests-cucumber/src/test/resources/dlab/endpoint.feature b/integration-tests-cucumber/src/test/resources/dlab/endpoint.feature
deleted file mode 100644
index 1f7fe14..0000000
--- a/integration-tests-cucumber/src/test/resources/dlab/endpoint.feature
+++ /dev/null
@@ -1,75 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-Feature: Endpoint management in DLab
- Such feature allowed to manage endpoint inside DLab
-
- Scenario Outline: Create new endpoint when it does not exist
-
- Given There is no endpoints in DLab
- And User try to create new endpoint with name "<name>" and uri "<uri>" and account "<account>" and "<tag>"
- When User send create new endpoint request
- Then Response status code is 200
- And Endpoint URI is present in location header
- Examples:
- | name | uri | account | tag |
- | test_endpoint | someuri | 123231312 | some_tag |
-
-
- Scenario Outline: Create new endpoint when it exist already
-
- Given There is no endpoints in DLab
- And User try to create new endpoint with name "<name>" and uri "<uri>" and account "<account>" and "<tag>"
- And User send create new endpoint request
- When User try to create new endpoint with name "<name>" and uri "<uri>" and account "<account>" and "<tag>"
- And User send create new endpoint request
- Then Response status code is 409
- Examples:
- | name | uri | account | tag |
- | test_endpoint | someuri | 123231312 | some_tag |
-
-
- Scenario Outline: Get information for endpoint
-
- Given There is no endpoints in DLab
- And User try to create new endpoint with name "<name>" and uri "<uri>" and account "<account>" and "<tag>"
- And User send create new endpoint request
- When User try to get information about endpoint with name "<name>"
- Then Response status code is 200
- And Endpoint information is successfully returned with name "<name>", uri "<uri>", account "<account>", and tag "<tag>"
- Examples:
- | name | uri | account | tag |
- | test_endpoint | someuri | 123231312 | some_tag |
-
-
- Scenario: Get list of endpoints
-
- Given There is no endpoints in DLab
- And User try to create new endpoint with name "test1" and uri "someuri1" and account "123" and "customTag1"
- And User send create new endpoint request
- And User try to create new endpoint with name "test2" and uri "someuri2" and account "1233" and "customTag4"
- And User send create new endpoint request
- When User try to get information about endpoints
- Then Response status code is 200
- And There are endpoints with name test1 and test2
-
- Scenario: Get not endpoint that does not exist
-
- Given There is no endpoints in DLab
- When User try to get information about endpoint with name "someName"
- Then Response status code is 404
diff --git a/integration-tests-cucumber/src/test/resources/dlab/login.feature b/integration-tests-cucumber/src/test/resources/dlab/login.feature
deleted file mode 100644
index 1675aad..0000000
--- a/integration-tests-cucumber/src/test/resources/dlab/login.feature
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-Feature: DLab login API
- Used to check DLab login flow
-
- Scenario Outline: User try to login to DLab
- Given User try to login to Dlab with "<username>" and "<password>"
- When user try to login
- Then response code is "<status>"
-
- Examples:
- | username | password | status |
- | test | pass | 200 |
- | not_valid_user | pass | 401 |
\ No newline at end of file
diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml
index 9ac5284..5b3e753 100644
--- a/integration-tests/pom.xml
+++ b/integration-tests/pom.xml
@@ -180,7 +180,7 @@
</dependencies>
<build>
- <finalName>${project.artifactId}-${dlab.version}</finalName>
+ <finalName>${project.artifactId}-${datalab.version}</finalName>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
@@ -188,7 +188,7 @@
<configuration>
<source>${java.version}</source>
<target>${java.version}</target>
-
+
</configuration>
</plugin>
@@ -248,17 +248,17 @@
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<!-- <mainClass>com.epam.dlab.automation</mainClass> -->
- <manifestEntries>
- <Created-By><EPAM> Systems</Created-By>
- <Name>com/epam/dlab/automation</Name>
- <Implementation-Title>DLab Integration Tests</Implementation-Title>
- <Implementation-Version>${dlab.version}</Implementation-Version>
- <Implementation-Vendor><EPAM> Systems</Implementation-Vendor>
- <Build-Time>${maven.build.timestamp}</Build-Time>
- <Build-OS>${os.name}</Build-OS>
- <GIT-Branch>${scmBranch}</GIT-Branch>
- <GIT-Commit>${buildNumber}</GIT-Commit>
- </manifestEntries>
+ <manifestEntries>
+ <Created-By><EPAM> Systems</Created-By>
+ <Name>com/epam/dlab/automation</Name>
+ <Implementation-Title>DLab Integration Tests</Implementation-Title>
+ <Implementation-Version>${datalab.version}</Implementation-Version>
+ <Implementation-Vendor><EPAM> Systems</Implementation-Vendor>
+ <Build-Time>${maven.build.timestamp}</Build-Time>
+ <Build-OS>${os.name}</Build-OS>
+ <GIT-Branch>${scmBranch}</GIT-Branch>
+ <GIT-Commit>${buildNumber}</GIT-Commit>
+ </manifestEntries>
</transformer>
</transformers>
<filters>
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/cloud/VirtualMachineStatusChecker.java b/integration-tests/src/main/java/com/epam/dlab/automation/cloud/VirtualMachineStatusChecker.java
index 0ecff1d..79313b3 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/cloud/VirtualMachineStatusChecker.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/cloud/VirtualMachineStatusChecker.java
@@ -17,15 +17,15 @@
* under the License.
*/
-package com.epam.dlab.automation.cloud;
-
-import com.epam.dlab.automation.cloud.aws.AmazonHelper;
-import com.epam.dlab.automation.cloud.aws.AmazonInstanceState;
-import com.epam.dlab.automation.cloud.azure.AzureHelper;
-import com.epam.dlab.automation.cloud.gcp.GcpHelper;
-import com.epam.dlab.automation.cloud.gcp.GcpInstanceState;
-import com.epam.dlab.automation.helper.CloudProvider;
-import com.epam.dlab.automation.helper.ConfigPropertyValue;
+package com.epam.datalab.automation.cloud;
+
+import com.epam.datalab.automation.cloud.aws.AmazonHelper;
+import com.epam.datalab.automation.cloud.aws.AmazonInstanceState;
+import com.epam.datalab.automation.cloud.azure.AzureHelper;
+import com.epam.datalab.automation.cloud.gcp.GcpHelper;
+import com.epam.datalab.automation.cloud.gcp.GcpInstanceState;
+import com.epam.datalab.automation.helper.CloudProvider;
+import com.epam.datalab.automation.helper.ConfigPropertyValue;
import com.microsoft.azure.management.compute.PowerState;
import org.testng.Assert;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/cloud/aws/AmazonHelper.java b/integration-tests/src/main/java/com/epam/dlab/automation/cloud/aws/AmazonHelper.java
index 117f3ee..c5cc69f 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/cloud/aws/AmazonHelper.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/cloud/aws/AmazonHelper.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.cloud.aws;
+package com.epam.datalab.automation.cloud.aws;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
@@ -30,9 +30,9 @@ import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.AccessControlList;
import com.amazonaws.services.s3.model.Grant;
-import com.epam.dlab.automation.exceptions.CloudException;
-import com.epam.dlab.automation.helper.ConfigPropertyValue;
-import com.epam.dlab.automation.helper.NamingHelper;
+import com.epam.datalab.automation.exceptions.CloudException;
+import com.epam.datalab.automation.helper.ConfigPropertyValue;
+import com.epam.datalab.automation.helper.NamingHelper;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.testng.Assert;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/cloud/aws/AmazonInstanceState.java b/integration-tests/src/main/java/com/epam/dlab/automation/cloud/aws/AmazonInstanceState.java
index 576be9d..2d49bec 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/cloud/aws/AmazonInstanceState.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/cloud/aws/AmazonInstanceState.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.cloud.aws;
+package com.epam.datalab.automation.cloud.aws;
public enum AmazonInstanceState {
STARTING,
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/cloud/azure/AzureHelper.java b/integration-tests/src/main/java/com/epam/dlab/automation/cloud/azure/AzureHelper.java
index 25fb388..79c6654 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/cloud/azure/AzureHelper.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/cloud/azure/AzureHelper.java
@@ -17,10 +17,10 @@
* under the License.
*/
-package com.epam.dlab.automation.cloud.azure;
+package com.epam.datalab.automation.cloud.azure;
-import com.epam.dlab.automation.exceptions.CloudException;
-import com.epam.dlab.automation.helper.ConfigPropertyValue;
+import com.epam.datalab.automation.exceptions.CloudException;
+import com.epam.datalab.automation.helper.ConfigPropertyValue;
import com.microsoft.azure.management.Azure;
import com.microsoft.azure.management.compute.PowerState;
import com.microsoft.azure.management.compute.VirtualMachine;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/cloud/gcp/GcpHelper.java b/integration-tests/src/main/java/com/epam/dlab/automation/cloud/gcp/GcpHelper.java
index 7240464..b796510 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/cloud/gcp/GcpHelper.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/cloud/gcp/GcpHelper.java
@@ -17,10 +17,10 @@
* under the License.
*/
-package com.epam.dlab.automation.cloud.gcp;
+package com.epam.datalab.automation.cloud.gcp;
-import com.epam.dlab.automation.exceptions.CloudException;
-import com.epam.dlab.automation.helper.ConfigPropertyValue;
+import com.epam.datalab.automation.exceptions.CloudException;
+import com.epam.datalab.automation.helper.ConfigPropertyValue;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.HttpTransport;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/cloud/gcp/GcpInstanceState.java b/integration-tests/src/main/java/com/epam/dlab/automation/cloud/gcp/GcpInstanceState.java
index 5c084c1..7d0207a 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/cloud/gcp/GcpInstanceState.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/cloud/gcp/GcpInstanceState.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.cloud.gcp;
+package com.epam.datalab.automation.cloud.gcp;
public enum GcpInstanceState {
STARTING,
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/docker/AckStatus.java b/integration-tests/src/main/java/com/epam/dlab/automation/docker/AckStatus.java
index df4b7e7..16dee01 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/docker/AckStatus.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/docker/AckStatus.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.docker;
+package com.epam.datalab.automation.docker;
public class AckStatus {
private int status;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/docker/Bridge.java b/integration-tests/src/main/java/com/epam/dlab/automation/docker/Bridge.java
index 8cc6015..5a7ef0e 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/docker/Bridge.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/docker/Bridge.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.docker;
+package com.epam.datalab.automation.docker;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/docker/Docker.java b/integration-tests/src/main/java/com/epam/dlab/automation/docker/Docker.java
index 012edaf..3d52598 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/docker/Docker.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/docker/Docker.java
@@ -17,10 +17,10 @@
* under the License.
*/
-package com.epam.dlab.automation.docker;
+package com.epam.datalab.automation.docker;
-import com.epam.dlab.automation.exceptions.DockerException;
-import com.epam.dlab.automation.helper.ConfigPropertyValue;
+import com.epam.datalab.automation.exceptions.DockerException;
+import com.epam.datalab.automation.helper.ConfigPropertyValue;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jcraft.jsch.ChannelExec;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/docker/DockerContainer.java b/integration-tests/src/main/java/com/epam/dlab/automation/docker/DockerContainer.java
index c22a688..f0c0c9e 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/docker/DockerContainer.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/docker/DockerContainer.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.docker;
+package com.epam.datalab.automation.docker;
import com.fasterxml.jackson.annotation.JsonProperty;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/docker/HostConfig.java b/integration-tests/src/main/java/com/epam/dlab/automation/docker/HostConfig.java
index c2d5db0..a5a0adb 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/docker/HostConfig.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/docker/HostConfig.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.docker;
+package com.epam.datalab.automation.docker;
import com.fasterxml.jackson.annotation.JsonProperty;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/docker/Labels.java b/integration-tests/src/main/java/com/epam/dlab/automation/docker/Labels.java
index 1e49a60..adbfe31 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/docker/Labels.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/docker/Labels.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.docker;
+package com.epam.datalab.automation.docker;
class Labels {
}
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/docker/NetworkSettings.java b/integration-tests/src/main/java/com/epam/dlab/automation/docker/NetworkSettings.java
index 295c217..d21e52e 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/docker/NetworkSettings.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/docker/NetworkSettings.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.docker;
+package com.epam.datalab.automation.docker;
import com.fasterxml.jackson.annotation.JsonProperty;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/docker/Networks.java b/integration-tests/src/main/java/com/epam/dlab/automation/docker/Networks.java
index 2679fa3..2ea6542 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/docker/Networks.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/docker/Networks.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.docker;
+package com.epam.datalab.automation.docker;
public class Networks {
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/docker/SSHConnect.java b/integration-tests/src/main/java/com/epam/dlab/automation/docker/SSHConnect.java
index fa369fa..6d498f3 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/docker/SSHConnect.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/docker/SSHConnect.java
@@ -17,9 +17,9 @@
* under the License.
*/
-package com.epam.dlab.automation.docker;
+package com.epam.datalab.automation.docker;
-import com.epam.dlab.automation.helper.ConfigPropertyValue;
+import com.epam.datalab.automation.helper.ConfigPropertyValue;
import com.jcraft.jsch.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/CloudException.java b/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/CloudException.java
index b576931..f440c39 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/CloudException.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/CloudException.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.exceptions;
+package com.epam.datalab.automation.exceptions;
public class CloudException extends RuntimeException {
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/DockerException.java b/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/DockerException.java
index ae8d7df..69b6f13 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/DockerException.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/DockerException.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.exceptions;
+package com.epam.datalab.automation.exceptions;
public class DockerException extends RuntimeException {
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/JenkinsException.java b/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/JenkinsException.java
index 4b70836..0171ce9 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/JenkinsException.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/JenkinsException.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.exceptions;
+package com.epam.datalab.automation.exceptions;
public class JenkinsException extends RuntimeException {
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/LoadFailException.java b/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/LoadFailException.java
index 16d4f20..875e9f1 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/LoadFailException.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/exceptions/LoadFailException.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.exceptions;
+package com.epam.datalab.automation.exceptions;
public class LoadFailException extends RuntimeException {
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/helper/CloudHelper.java b/integration-tests/src/main/java/com/epam/dlab/automation/helper/CloudHelper.java
index 4e0894f..3917764 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/helper/CloudHelper.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/helper/CloudHelper.java
@@ -17,18 +17,18 @@
* under the License.
*/
-package com.epam.dlab.automation.helper;
+package com.epam.datalab.automation.helper;
import com.amazonaws.services.ec2.model.Instance;
import com.amazonaws.services.ec2.model.Tag;
-import com.epam.dlab.automation.cloud.aws.AmazonHelper;
-import com.epam.dlab.automation.cloud.azure.AzureHelper;
-import com.epam.dlab.automation.cloud.gcp.GcpHelper;
-import com.epam.dlab.automation.exceptions.CloudException;
-import com.epam.dlab.automation.model.DeployClusterDto;
-import com.epam.dlab.automation.model.DeployDataProcDto;
-import com.epam.dlab.automation.model.DeployEMRDto;
-import com.epam.dlab.automation.model.NotebookConfig;
+import com.epam.datalab.automation.cloud.aws.AmazonHelper;
+import com.epam.datalab.automation.cloud.azure.AzureHelper;
+import com.epam.datalab.automation.cloud.gcp.GcpHelper;
+import com.epam.datalab.automation.exceptions.CloudException;
+import com.epam.datalab.automation.model.DeployClusterDto;
+import com.epam.datalab.automation.model.DeployDataProcDto;
+import com.epam.datalab.automation.model.DeployEMRDto;
+import com.epam.datalab.automation.model.NotebookConfig;
import org.apache.commons.lang3.StringUtils;
import java.io.IOException;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/helper/CloudProvider.java b/integration-tests/src/main/java/com/epam/dlab/automation/helper/CloudProvider.java
index f5241a4..23163f6 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/helper/CloudProvider.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/helper/CloudProvider.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.helper;
+package com.epam.datalab.automation.helper;
public class CloudProvider {
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/helper/ConfigPropertyValue.java b/integration-tests/src/main/java/com/epam/dlab/automation/helper/ConfigPropertyValue.java
index aeb6036..01e0cdf 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/helper/ConfigPropertyValue.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/helper/ConfigPropertyValue.java
@@ -17,9 +17,9 @@
* under the License.
*/
-package com.epam.dlab.automation.helper;
+package com.epam.datalab.automation.helper;
-import com.epam.dlab.automation.exceptions.LoadFailException;
+import com.epam.datalab.automation.exceptions.LoadFailException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/helper/NamingHelper.java b/integration-tests/src/main/java/com/epam/dlab/automation/helper/NamingHelper.java
index 3094f1c..44fe86e 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/helper/NamingHelper.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/helper/NamingHelper.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.helper;
+package com.epam.datalab.automation.helper;
import java.io.IOException;
import java.text.SimpleDateFormat;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/helper/PropertiesResolver.java b/integration-tests/src/main/java/com/epam/dlab/automation/helper/PropertiesResolver.java
index f71ccc3..eeabe1b 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/helper/PropertiesResolver.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/helper/PropertiesResolver.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.helper;
+package com.epam.datalab.automation.helper;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/helper/WaitForStatus.java b/integration-tests/src/main/java/com/epam/dlab/automation/helper/WaitForStatus.java
index 4cb4129..bccbef0 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/helper/WaitForStatus.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/helper/WaitForStatus.java
@@ -17,11 +17,11 @@
* under the License.
*/
-package com.epam.dlab.automation.helper;
+package com.epam.datalab.automation.helper;
-import com.epam.dlab.automation.http.ContentType;
-import com.epam.dlab.automation.http.HttpRequest;
-import com.epam.dlab.automation.http.HttpStatusCode;
+import com.epam.datalab.automation.http.ContentType;
+import com.epam.datalab.automation.http.HttpRequest;
+import com.epam.datalab.automation.http.HttpStatusCode;
import com.jayway.restassured.path.json.JsonPath;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/http/ApiPath.java b/integration-tests/src/main/java/com/epam/dlab/automation/http/ApiPath.java
index c3dc9f1..936f464 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/http/ApiPath.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/http/ApiPath.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.http;
+package com.epam.datalab.automation.http;
public class ApiPath {
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/http/ContentType.java b/integration-tests/src/main/java/com/epam/dlab/automation/http/ContentType.java
index e2a482c..93fe54b 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/http/ContentType.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/http/ContentType.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.http;
+package com.epam.datalab.automation.http;
public class ContentType{
public static final String FORMDATA = "multipart/form-data";
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/http/HttpRequest.java b/integration-tests/src/main/java/com/epam/dlab/automation/http/HttpRequest.java
index 2ef936f..0c9530d 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/http/HttpRequest.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/http/HttpRequest.java
@@ -17,9 +17,9 @@
* under the License.
*/
-package com.epam.dlab.automation.http;
+package com.epam.datalab.automation.http;
-import com.epam.dlab.automation.helper.ConfigPropertyValue;
+import com.epam.datalab.automation.helper.ConfigPropertyValue;
import com.jayway.restassured.http.ContentType;
import com.jayway.restassured.response.Response;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/http/HttpStatusCode.java b/integration-tests/src/main/java/com/epam/dlab/automation/http/HttpStatusCode.java
index 6c4aef0..e6e5dad 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/http/HttpStatusCode.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/http/HttpStatusCode.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.http;
+package com.epam.datalab.automation.http;
public class HttpStatusCode {
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsConfigProperties.java b/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsConfigProperties.java
index 158715b..9240ea2 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsConfigProperties.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsConfigProperties.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.jenkins;
+package com.epam.datalab.automation.jenkins;
public class JenkinsConfigProperties {
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsResponseElements.java b/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsResponseElements.java
index 0aca9f6..989558e 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsResponseElements.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsResponseElements.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.jenkins;
+package com.epam.datalab.automation.jenkins;
public class JenkinsResponseElements {
public static final String IN_QUEUE_ELEMENT = "freeStyleProject.inQueue";
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsService.java b/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsService.java
index 5d2a995..4606f55 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsService.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsService.java
@@ -17,12 +17,12 @@
* under the License.
*/
-package com.epam.dlab.automation.jenkins;
+package com.epam.datalab.automation.jenkins;
-import com.epam.dlab.automation.exceptions.JenkinsException;
-import com.epam.dlab.automation.helper.ConfigPropertyValue;
-import com.epam.dlab.automation.helper.NamingHelper;
-import com.epam.dlab.automation.http.HttpStatusCode;
+import com.epam.datalab.automation.exceptions.JenkinsException;
+import com.epam.datalab.automation.helper.ConfigPropertyValue;
+import com.epam.datalab.automation.helper.NamingHelper;
+import com.epam.datalab.automation.http.HttpStatusCode;
import com.jayway.restassured.RestAssured;
import com.jayway.restassured.authentication.FormAuthConfig;
import com.jayway.restassured.http.ContentType;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsUrls.java b/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsUrls.java
index 04bedcc..3e88c55 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsUrls.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/jenkins/JenkinsUrls.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.jenkins;
+package com.epam.datalab.automation.jenkins;
public class JenkinsUrls {
public static final String API = "api/xml";
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/model/CreateNotebookDto.java b/integration-tests/src/main/java/com/epam/dlab/automation/model/CreateNotebookDto.java
index 71bfedf..fc309ba 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/model/CreateNotebookDto.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/model/CreateNotebookDto.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.model;
+package com.epam.datalab.automation.model;
import com.fasterxml.jackson.annotation.JsonProperty;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/model/DeployClusterDto.java b/integration-tests/src/main/java/com/epam/dlab/automation/model/DeployClusterDto.java
index 695a5eb..1282972 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/model/DeployClusterDto.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/model/DeployClusterDto.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.model;
+package com.epam.datalab.automation.model;
import com.fasterxml.jackson.annotation.JsonProperty;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/model/DeployDataProcDto.java b/integration-tests/src/main/java/com/epam/dlab/automation/model/DeployDataProcDto.java
index b3b64e2..5448a6c 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/model/DeployDataProcDto.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/model/DeployDataProcDto.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.model;
+package com.epam.datalab.automation.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/model/DeployEMRDto.java b/integration-tests/src/main/java/com/epam/dlab/automation/model/DeployEMRDto.java
index 8f3ac1e..2e43a2b 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/model/DeployEMRDto.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/model/DeployEMRDto.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.model;
+package com.epam.datalab.automation.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/model/DeploySparkDto.java b/integration-tests/src/main/java/com/epam/dlab/automation/model/DeploySparkDto.java
index d1b4734..445880f 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/model/DeploySparkDto.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/model/DeploySparkDto.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.model;
+package com.epam.datalab.automation.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/model/ExploratoryImageDto.java b/integration-tests/src/main/java/com/epam/dlab/automation/model/ExploratoryImageDto.java
index 84950df..72e5a90 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/model/ExploratoryImageDto.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/model/ExploratoryImageDto.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.model;
+package com.epam.datalab.automation.model;
import com.fasterxml.jackson.annotation.JsonProperty;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/model/ImageDto.java b/integration-tests/src/main/java/com/epam/dlab/automation/model/ImageDto.java
index 0a5dc0b..3d7a0df 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/model/ImageDto.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/model/ImageDto.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.model;
+package com.epam.datalab.automation.model;
public class ImageDto {
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/model/JsonMapperDto.java b/integration-tests/src/main/java/com/epam/dlab/automation/model/JsonMapperDto.java
index 11cdd50..9f38008 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/model/JsonMapperDto.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/model/JsonMapperDto.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.model;
+package com.epam.datalab.automation.model;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.CollectionType;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/model/Lib.java b/integration-tests/src/main/java/com/epam/dlab/automation/model/Lib.java
index a804c18..d86adff 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/model/Lib.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/model/Lib.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.model;
+package com.epam.datalab.automation.model;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/model/LoginDto.java b/integration-tests/src/main/java/com/epam/dlab/automation/model/LoginDto.java
index 4018643..6130a02 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/model/LoginDto.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/model/LoginDto.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.model;
+package com.epam.datalab.automation.model;
import com.fasterxml.jackson.annotation.JsonProperty;
diff --git a/integration-tests/src/main/java/com/epam/dlab/automation/model/NotebookConfig.java b/integration-tests/src/main/java/com/epam/dlab/automation/model/NotebookConfig.java
index bac7893..a788f87 100644
--- a/integration-tests/src/main/java/com/epam/dlab/automation/model/NotebookConfig.java
+++ b/integration-tests/src/main/java/com/epam/dlab/automation/model/NotebookConfig.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.model;
+package com.epam.datalab.automation.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects;
diff --git a/integration-tests/src/main/resources/log4j2.xml b/integration-tests/src/main/resources/log4j2.xml
index 8c91840..02f1c6f 100644
--- a/integration-tests/src/main/resources/log4j2.xml
+++ b/integration-tests/src/main/resources/log4j2.xml
@@ -50,14 +50,14 @@
<Loggers>
- <Root level="info">
- <AppenderRef ref="file" />
- <AppenderRef ref="console" />
- </Root>
- <Logger name="com.epam.dlab.automation" level="debug" additivity="false">
- <AppenderRef ref="file" />
- <AppenderRef ref="console" />
- </Logger>
- </Loggers>
+ <Root level="info">
+ <AppenderRef ref="file"/>
+ <AppenderRef ref="console"/>
+ </Root>
+ <Logger name="com.epam.datalab.automation" level="debug" additivity="false">
+ <AppenderRef ref="file"/>
+ <AppenderRef ref="console"/>
+ </Logger>
+ </Loggers>
</Configuration>
\ No newline at end of file
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/TestCallable.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/TestCallable.java
index 881b69e..de0de05 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/TestCallable.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/TestCallable.java
@@ -17,22 +17,22 @@
* under the License.
*/
-package com.epam.dlab.automation.test;
-
-import com.epam.dlab.automation.cloud.VirtualMachineStatusChecker;
-import com.epam.dlab.automation.cloud.aws.AmazonHelper;
-import com.epam.dlab.automation.docker.Docker;
-import com.epam.dlab.automation.helper.*;
-import com.epam.dlab.automation.http.ApiPath;
-import com.epam.dlab.automation.http.ContentType;
-import com.epam.dlab.automation.http.HttpRequest;
-import com.epam.dlab.automation.http.HttpStatusCode;
-import com.epam.dlab.automation.model.*;
-import com.epam.dlab.automation.test.libs.LibsHelper;
-import com.epam.dlab.automation.test.libs.TestLibGroupStep;
-import com.epam.dlab.automation.test.libs.TestLibInstallStep;
-import com.epam.dlab.automation.test.libs.TestLibListStep;
-import com.epam.dlab.automation.test.libs.models.LibToSearchData;
+package com.epam.datalab.automation.test;
+
+import com.epam.datalab.automation.cloud.VirtualMachineStatusChecker;
+import com.epam.datalab.automation.cloud.aws.AmazonHelper;
+import com.epam.datalab.automation.docker.Docker;
+import com.epam.datalab.automation.helper.*;
+import com.epam.datalab.automation.http.ApiPath;
+import com.epam.datalab.automation.http.ContentType;
+import com.epam.datalab.automation.http.HttpRequest;
+import com.epam.datalab.automation.http.HttpStatusCode;
+import com.epam.datalab.automation.model.*;
+import com.epam.datalab.automation.test.libs.LibsHelper;
+import com.epam.datalab.automation.test.libs.TestLibGroupStep;
+import com.epam.datalab.automation.test.libs.TestLibInstallStep;
+import com.epam.datalab.automation.test.libs.TestLibListStep;
+import com.epam.datalab.automation.test.libs.models.LibToSearchData;
import com.jayway.restassured.response.Response;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/TestDataEngineService.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/TestDataEngineService.java
index ad73842..f828a75 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/TestDataEngineService.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/TestDataEngineService.java
@@ -17,14 +17,14 @@
* under the License.
*/
-package com.epam.dlab.automation.test;
-
-import com.epam.dlab.automation.docker.AckStatus;
-import com.epam.dlab.automation.docker.SSHConnect;
-import com.epam.dlab.automation.helper.CloudHelper;
-import com.epam.dlab.automation.helper.ConfigPropertyValue;
-import com.epam.dlab.automation.helper.NamingHelper;
-import com.epam.dlab.automation.helper.PropertiesResolver;
+package com.epam.datalab.automation.test;
+
+import com.epam.datalab.automation.docker.AckStatus;
+import com.epam.datalab.automation.docker.SSHConnect;
+import com.epam.datalab.automation.helper.CloudHelper;
+import com.epam.datalab.automation.helper.ConfigPropertyValue;
+import com.epam.datalab.automation.helper.NamingHelper;
+import com.epam.datalab.automation.helper.PropertiesResolver;
import com.jcraft.jsch.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/TestServices.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/TestServices.java
index 8d9aad4..801bee1 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/TestServices.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/TestServices.java
@@ -17,19 +17,19 @@
* under the License.
*/
-package com.epam.dlab.automation.test;
-
-import com.epam.dlab.automation.cloud.VirtualMachineStatusChecker;
-import com.epam.dlab.automation.docker.Docker;
-import com.epam.dlab.automation.helper.*;
-import com.epam.dlab.automation.http.ApiPath;
-import com.epam.dlab.automation.http.ContentType;
-import com.epam.dlab.automation.http.HttpRequest;
-import com.epam.dlab.automation.http.HttpStatusCode;
-import com.epam.dlab.automation.jenkins.JenkinsService;
-import com.epam.dlab.automation.model.Lib;
-import com.epam.dlab.automation.model.LoginDto;
-import com.epam.dlab.automation.model.NotebookConfig;
+package com.epam.datalab.automation.test;
+
+import com.epam.datalab.automation.cloud.VirtualMachineStatusChecker;
+import com.epam.datalab.automation.docker.Docker;
+import com.epam.datalab.automation.helper.*;
+import com.epam.datalab.automation.http.ApiPath;
+import com.epam.datalab.automation.http.ContentType;
+import com.epam.datalab.automation.http.HttpRequest;
+import com.epam.datalab.automation.http.HttpStatusCode;
+import com.epam.datalab.automation.jenkins.JenkinsService;
+import com.epam.datalab.automation.model.Lib;
+import com.epam.datalab.automation.model.LoginDto;
+import com.epam.datalab.automation.model.NotebookConfig;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.restassured.RestAssured;
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/LibraryNotFoundException.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/LibraryNotFoundException.java
index b8fca93..c6d8df1 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/LibraryNotFoundException.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/LibraryNotFoundException.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.test.libs;
+package com.epam.datalab.automation.test.libs;
class LibraryNotFoundException extends RuntimeException {
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/LibsHelper.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/LibsHelper.java
index 471679c..70fa2b7 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/LibsHelper.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/LibsHelper.java
@@ -17,11 +17,11 @@
* under the License.
*/
-package com.epam.dlab.automation.test.libs;
+package com.epam.datalab.automation.test.libs;
-import com.epam.dlab.automation.helper.NamingHelper;
+import com.epam.datalab.automation.helper.NamingHelper;
-import static com.epam.dlab.automation.helper.NamingHelper.*;
+import static com.epam.datalab.automation.helper.NamingHelper.*;
public class LibsHelper {
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestDescription.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestDescription.java
index 5c156b4..dfe5b22 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestDescription.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestDescription.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.test.libs;
+package com.epam.datalab.automation.test.libs;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibGroupStep.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibGroupStep.java
index 57d56d4..afecccf 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibGroupStep.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibGroupStep.java
@@ -17,13 +17,13 @@
* under the License.
*/
-package com.epam.dlab.automation.test.libs;
+package com.epam.datalab.automation.test.libs;
-import com.epam.dlab.automation.helper.ConfigPropertyValue;
-import com.epam.dlab.automation.helper.NamingHelper;
-import com.epam.dlab.automation.http.HttpRequest;
-import com.epam.dlab.automation.http.HttpStatusCode;
-import com.epam.dlab.automation.model.JsonMapperDto;
+import com.epam.datalab.automation.helper.ConfigPropertyValue;
+import com.epam.datalab.automation.helper.NamingHelper;
+import com.epam.datalab.automation.http.HttpRequest;
+import com.epam.datalab.automation.http.HttpStatusCode;
+import com.epam.datalab.automation.model.JsonMapperDto;
import com.jayway.restassured.response.Response;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibInstallStep.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibInstallStep.java
index 9b9d521..7fa95e7 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibInstallStep.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibInstallStep.java
@@ -17,17 +17,17 @@
* under the License.
*/
-package com.epam.dlab.automation.test.libs;
-
-import com.epam.dlab.automation.helper.ConfigPropertyValue;
-import com.epam.dlab.automation.helper.NamingHelper;
-import com.epam.dlab.automation.http.ContentType;
-import com.epam.dlab.automation.http.HttpRequest;
-import com.epam.dlab.automation.http.HttpStatusCode;
-import com.epam.dlab.automation.model.Lib;
-import com.epam.dlab.automation.test.libs.models.LibInstallRequest;
-import com.epam.dlab.automation.test.libs.models.LibStatusResponse;
-import com.epam.dlab.automation.test.libs.models.LibraryStatus;
+package com.epam.datalab.automation.test.libs;
+
+import com.epam.datalab.automation.helper.ConfigPropertyValue;
+import com.epam.datalab.automation.helper.NamingHelper;
+import com.epam.datalab.automation.http.ContentType;
+import com.epam.datalab.automation.http.HttpRequest;
+import com.epam.datalab.automation.http.HttpStatusCode;
+import com.epam.datalab.automation.model.Lib;
+import com.epam.datalab.automation.test.libs.models.LibInstallRequest;
+import com.epam.datalab.automation.test.libs.models.LibStatusResponse;
+import com.epam.datalab.automation.test.libs.models.LibraryStatus;
import com.jayway.restassured.response.Response;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibListStep.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibListStep.java
index 89566c2..67d1138 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibListStep.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibListStep.java
@@ -17,16 +17,16 @@
* under the License.
*/
-package com.epam.dlab.automation.test.libs;
-
-import com.epam.dlab.automation.helper.ConfigPropertyValue;
-import com.epam.dlab.automation.helper.NamingHelper;
-import com.epam.dlab.automation.http.ContentType;
-import com.epam.dlab.automation.http.HttpRequest;
-import com.epam.dlab.automation.http.HttpStatusCode;
-import com.epam.dlab.automation.model.Lib;
-import com.epam.dlab.automation.test.libs.models.LibSearchRequest;
-import com.epam.dlab.automation.test.libs.models.LibToSearchData;
+package com.epam.datalab.automation.test.libs;
+
+import com.epam.datalab.automation.helper.ConfigPropertyValue;
+import com.epam.datalab.automation.helper.NamingHelper;
+import com.epam.datalab.automation.http.ContentType;
+import com.epam.datalab.automation.http.HttpRequest;
+import com.epam.datalab.automation.http.HttpStatusCode;
+import com.epam.datalab.automation.model.Lib;
+import com.epam.datalab.automation.test.libs.models.LibSearchRequest;
+import com.epam.datalab.automation.test.libs.models.LibToSearchData;
import com.jayway.restassured.response.Response;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibStep.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibStep.java
index 5930f77..b93b1bc 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibStep.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/TestLibStep.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.test.libs;
+package com.epam.datalab.automation.test.libs;
import java.lang.annotation.Annotation;
import java.util.concurrent.TimeUnit;
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibInstallRequest.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibInstallRequest.java
index ad48b07..6898d31 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibInstallRequest.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibInstallRequest.java
@@ -17,9 +17,9 @@
* under the License.
*/
-package com.epam.dlab.automation.test.libs.models;
+package com.epam.datalab.automation.test.libs.models;
-import com.epam.dlab.automation.model.Lib;
+import com.epam.datalab.automation.model.Lib;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects;
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibSearchRequest.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibSearchRequest.java
index 45ffa32..21b5ff5 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibSearchRequest.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibSearchRequest.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.test.libs.models;
+package com.epam.datalab.automation.test.libs.models;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects;
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibStatusResponse.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibStatusResponse.java
index cf79d82..a5671f7 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibStatusResponse.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibStatusResponse.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.test.libs.models;
+package com.epam.datalab.automation.test.libs.models;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibToSearchData.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibToSearchData.java
index e6aa205..58849eb 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibToSearchData.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibToSearchData.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.test.libs.models;
+package com.epam.datalab.automation.test.libs.models;
import com.fasterxml.jackson.annotation.JsonProperty;
diff --git a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibraryStatus.java b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibraryStatus.java
index 1be3139..0fc8308 100644
--- a/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibraryStatus.java
+++ b/integration-tests/src/test/java/com/epam/dlab/automation/test/libs/models/LibraryStatus.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package com.epam.dlab.automation.test.libs.models;
+package com.epam.datalab.automation.test.libs.models;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects;
diff --git a/integration-tests/src/test/resources/log4j2.xml b/integration-tests/src/test/resources/log4j2.xml
index 91d23a2..c170968 100644
--- a/integration-tests/src/test/resources/log4j2.xml
+++ b/integration-tests/src/test/resources/log4j2.xml
@@ -50,14 +50,14 @@
<Loggers>
- <Root level="info">
- <AppenderRef ref="file" />
- <AppenderRef ref="console" />
- </Root>
- <Logger name="com.epam.dlab.automation" level="debug" additivity="false">
- <AppenderRef ref="file" />
- <AppenderRef ref="console" />
- </Logger>
- </Loggers>
+ <Root level="info">
+ <AppenderRef ref="file"/>
+ <AppenderRef ref="console"/>
+ </Root>
+ <Logger name="com.epam.datalab.automation" level="debug" additivity="false">
+ <AppenderRef ref="file"/>
+ <AppenderRef ref="console"/>
+ </Logger>
+ </Loggers>
</Configuration>
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 95be206..f625c6b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,8 +22,8 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.epam.dlab</groupId>
- <artifactId>dlab</artifactId>
+ <groupId>com.epam.datalab</groupId>
+ <artifactId>datalab</artifactId>
<version>1.0</version>
<packaging>pom</packaging>
@@ -71,24 +71,24 @@
**/*Configuration.java,
**/*ApplicationConfiguration.java,
**/*Application.java,
- src/main/java/com/epam/dlab/dto/**,
- src/main/java/com/epam/dlab/backendapi/interceptor/**,
- src/main/java/com/epam/dlab/backendapi/auth/**,
- src/main/java/com/epam/dlab/backendapi/conf/**,
- src/main/java/com/epam/dlab/backendapi/domain/**,
- src/main/java/com/epam/dlab/backendapi/dropwizard/**,
- src/main/java/com/epam/dlab/backendapi/healthcheck/**,
- src/main/java/com/epam/dlab/backendapi/modules/**,
- src/main/java/com/epam/dlab/backendapi/resources/dto/**,
- src/main/java/com/epam/dlab/backendapi/roles/**,
- src/main/java/com/epam/dlab/backendapi/servlet/guacamole/**,
- src/main/java/com/epam/dlab/backendapi/util/**,
- src/main/java/com/epam/dlab/backendapi/validation/**,
- src/main/java/com/epam/dlab/backendapi/core/commands/**,
- src/main/java/com/epam/dlab/process/**,
- src/main/java/com/epam/dlab/backendapi/modules/**,
- src/main/java/com/epam/dlab/backendapi/validation/**,
- src/main/java/com/epam/dlab/backendapi/schedulers/**
+ src/main/java/com/epam/datalab/dto/**,
+ src/main/java/com/epam/datalab/backendapi/interceptor/**,
+ src/main/java/com/epam/datalab/backendapi/auth/**,
+ src/main/java/com/epam/datalab/backendapi/conf/**,
+ src/main/java/com/epam/datalab/backendapi/domain/**,
+ src/main/java/com/epam/datalab/backendapi/dropwizard/**,
+ src/main/java/com/epam/datalab/backendapi/healthcheck/**,
+ src/main/java/com/epam/datalab/backendapi/modules/**,
+ src/main/java/com/epam/datalab/backendapi/resources/dto/**,
+ src/main/java/com/epam/datalab/backendapi/roles/**,
+ src/main/java/com/epam/datalab/backendapi/servlet/guacamole/**,
+ src/main/java/com/epam/datalab/backendapi/util/**,
+ src/main/java/com/epam/datalab/backendapi/validation/**,
+ src/main/java/com/epam/datalab/backendapi/core/commands/**,
+ src/main/java/com/epam/datalab/process/**,
+ src/main/java/com/epam/datalab/backendapi/modules/**,
+ src/main/java/com/epam/datalab/backendapi/validation/**,
+ src/main/java/com/epam/datalab/backendapi/schedulers/**
</sonar.coverage.exclusions>
</properties>
</profile>
@@ -98,10 +98,10 @@
<module>services/common</module>
<module>services/provisioning-service</module>
<module>services/self-service</module>
- <module>services/dlab-model</module>
- <module>services/dlab-utils</module>
- <module>services/dlab-webapp-common</module>
- <module>services/dlab-mongo-migration</module>
+ <module>services/datalab-model</module>
+ <module>services/datalab-utils</module>
+ <module>services/datalab-webapp-common</module>
+ <module>services/datalab-mongo-migration</module>
<module>services/billing-azure</module>
<module>services/billing-gcp</module>
<module>services/billing-aws</module>
@@ -137,7 +137,7 @@
<version>1.3</version>
</dependency>
<dependency>
- <groupId>com.epam.dlab</groupId>
+ <groupId>com.epam.datalab</groupId>
<artifactId>common</artifactId>
<version>${project.version}</version>
</dependency>
@@ -191,7 +191,7 @@
</dependencies>
<build>
- <finalName>${project.artifactId}-${dlab.version}</finalName>
+ <finalName>${project.artifactId}-${datalab.version}</finalName>
<pluginManagement>
<plugins>
<plugin>
diff --git a/services/billing-aws/billing.yml b/services/billing-aws/billing.yml
index 3b1943f..f954797 100644
--- a/services/billing-aws/billing.yml
+++ b/services/billing-aws/billing.yml
@@ -30,7 +30,7 @@ host: MONGO_HOST
port: 27017
username: admin
password: MONGO_PASSWORD
-database: dlabdb
+database: datalabdb
# Adapter for reading source data. Known types: file, s3file
adapterIn:
@@ -44,13 +44,13 @@ adapterIn:
# Adapter for writing converted data. Known types: console, file, s3file, mongodb
adapterOut:
- - type: mongodlab
+ - type: mongodatalab
host: MONGO_HOST
port: 27017
username: admin
password: MONGO_PASSWORD
- database: dlabdb
-# bufferSize: 10000
+ database: datalabdb
+ # bufferSize: 10000
upsert: true
serviceBaseName: SERVICE_BASE_NAME
@@ -58,7 +58,7 @@ adapterOut:
filter:
- type: aws
currencyCode: USD
- columnDlabTag: CONF_BILLING_TAG
+ columnDatalabTag: CONF_BILLING_TAG
serviceBaseName: SERVICE_BASE_NAME
@@ -68,7 +68,7 @@ parser:
headerLineNo: 1
skipLines: 1
columnMapping: >-
- dlab_id=DLAB_ID;usage_date=USAGE_DATE;product=PRODUCT;
+ datalab_id=DATA_LAB_ID;usage_date=USAGE_DATE;product=PRODUCT;
usage_type=USAGE_TYPE;usage=USAGE;cost=COST;
resource_id=RESOURCE_ID;tags=TAGS
aggregate: day
@@ -88,7 +88,7 @@ logging:
appenders:
#- type: console
- type: file
- currentLogFilename: /var/opt/dlab/log/ssn/billing.log
+ currentLogFilename: /var/opt/datalab/log/ssn/billing.log
archive: true
- archivedLogFilenamePattern: /var/opt/dlab/log/ssn/billing-%d{yyyy-MM-dd}.log.gz
+ archivedLogFilenamePattern: /var/opt/datalab/log/ssn/billing-%d{yyyy-MM-dd}.log.gz
archivedFileCount: 10
\ No newline at end of file
diff --git a/services/billing-aws/pom.xml b/services/billing-aws/pom.xml
index ec4c830..30226f6 100644
--- a/services/billing-aws/pom.xml
+++ b/services/billing-aws/pom.xml
@@ -23,8 +23,8 @@
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.epam.dlab</groupId>
- <artifactId>dlab</artifactId>
+ <groupId>com.epam.datalab</groupId>
+ <artifactId>datalab</artifactId>
<version>1.0</version>
<relativePath>../../pom.xml</relativePath>
</parent>
@@ -58,7 +58,7 @@
<dependencies>
<dependency>
- <groupId>com.epam.dlab</groupId>
+ <groupId>com.epam.datalab</groupId>
<artifactId>common</artifactId>
</dependency>
@@ -173,8 +173,8 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.epam.dlab</groupId>
- <artifactId>dlab-model</artifactId>
+ <groupId>com.epam.datalab</groupId>
+ <artifactId>datalab-model</artifactId>
<version>${project.parent.version}</version>
</dependency>
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/BillingAwsApplication.java b/services/billing-aws/src/main/java/com/epam/datalab/BillingAwsApplication.java
new file mode 100644
index 0000000..c34ad40
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/BillingAwsApplication.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab;
+
+import com.epam.datalab.exceptions.InitializationException;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
+
+@SpringBootApplication
+@EnableMongoRepositories
+@EnableConfigurationProperties
+public class BillingAwsApplication {
+
+ public static void main(String[] args) throws InitializationException {
+ SpringApplication.run(BillingAwsApplication.class, args);
+ BillingServiceImpl.startApplication(args);
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/BillingService.java b/services/billing-aws/src/main/java/com/epam/datalab/BillingService.java
new file mode 100644
index 0000000..bfdd4c6
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/BillingService.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab;
+
+import com.epam.datalab.dto.billing.BillingData;
+
+import java.util.List;
+
+@FunctionalInterface
+public interface BillingService {
+ List<BillingData> getBillingData();
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/BillingServiceImpl.java b/services/billing-aws/src/main/java/com/epam/datalab/BillingServiceImpl.java
new file mode 100644
index 0000000..25e2fe5
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/BillingServiceImpl.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab;
+
+import com.epam.datalab.configuration.BillingToolConfiguration;
+import com.epam.datalab.configuration.BillingToolConfigurationFactory;
+import com.epam.datalab.core.parser.ParserBase;
+import com.epam.datalab.dto.billing.BillingData;
+import com.epam.datalab.exceptions.DatalabException;
+import com.epam.datalab.exceptions.InitializationException;
+import com.epam.datalab.util.ServiceUtils;
+import org.bson.Document;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+import java.time.LocalDate;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static com.epam.datalab.model.aws.ReportLine.FIELD_COST;
+import static com.epam.datalab.model.aws.ReportLine.FIELD_CURRENCY_CODE;
+import static com.epam.datalab.model.aws.ReportLine.FIELD_DATALAB_ID;
+import static com.epam.datalab.model.aws.ReportLine.FIELD_PRODUCT;
+import static com.epam.datalab.model.aws.ReportLine.FIELD_RESOURCE_TYPE;
+import static com.epam.datalab.model.aws.ReportLine.FIELD_USAGE_DATE;
+
+@Service
+public class BillingServiceImpl implements BillingService {
+ private static final Logger LOGGER = LoggerFactory.getLogger(BillingServiceImpl.class);
+ private static BillingToolConfiguration configuration;
+
+ public List<BillingData> getBillingData() {
+ try {
+ ParserBase parser = configuration.build();
+
+ List<BillingData> billingData = parser.parse()
+ .stream()
+ .map(this::toBillingData)
+ .collect(Collectors.toList());
+
+ if (!parser.getStatistics().isEmpty()) {
+ LOGGER.info("Billing report parser statistics:");
+ for (int i = 0; i < parser.getStatistics().size(); i++) {
+ LOGGER.info(" {}", parser.getStatistics().get(i).toString());
+ }
+ }
+
+ return billingData;
+ } catch (Exception e) {
+ LOGGER.error("Something went wrong ", e);
+ return Collections.emptyList();
+ }
+ }
+
+ private BillingData toBillingData(Document billingData) {
+ return BillingData.builder()
+ .tag(billingData.getString(FIELD_DATALAB_ID).toLowerCase())
+ .usageDateFrom(Optional.ofNullable(billingData.getString(FIELD_USAGE_DATE)).map(LocalDate::parse).orElse(null))
+ .usageDateTo(Optional.ofNullable(billingData.getString(FIELD_USAGE_DATE)).map(LocalDate::parse).orElse(null))
+ .usageDate(billingData.getString(FIELD_USAGE_DATE))
+ .product(billingData.getString(FIELD_PRODUCT))
+ .usageType(billingData.getString(FIELD_RESOURCE_TYPE))
+ .cost(billingData.getDouble(FIELD_COST))
+ .currency(billingData.getString(FIELD_CURRENCY_CODE))
+ .build();
+ }
+
+ public static void initialize(String filename) throws InitializationException {
+ LOGGER.debug("Billing report configuration file: {}", filename);
+ configuration = BillingToolConfigurationFactory.build(filename, BillingToolConfiguration.class);
+ }
+
+ public static void startApplication(String[] args) throws InitializationException {
+ if (ServiceUtils.printAppVersion(BillingTool.class, args)) {
+ return;
+ }
+
+ String confName = null;
+ for (int i = 0; i < args.length; i++) {
+ if (BillingTool.isKey("help", args[i])) {
+ i++;
+ Help.usage(i < args.length ? Arrays.copyOfRange(args, i, args.length) : null);
+ return;
+ } else if (BillingTool.isKey("conf", args[i])) {
+ i++;
+ if (i < args.length) {
+ confName = args[i];
+ } else {
+ throw new InitializationException("Missing the name of configuration file");
+ }
+ }
+ }
+
+ if (confName == null) {
+ Help.usage();
+ throw new InitializationException("Missing arguments");
+ }
+
+ BillingTool.setLoggerLevel();
+ try {
+ initialize(confName);
+ } catch (Exception e) {
+ throw new DatalabException("Billing scheduler failed", e);
+ }
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/BillingTool.java b/services/billing-aws/src/main/java/com/epam/datalab/BillingTool.java
new file mode 100644
index 0000000..f334883
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/BillingTool.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.LoggerContext;
+import com.epam.datalab.configuration.BillingToolConfiguration;
+import com.epam.datalab.configuration.BillingToolConfigurationFactory;
+import com.epam.datalab.core.parser.ParserBase;
+import com.epam.datalab.exceptions.AdapterException;
+import com.epam.datalab.exceptions.DatalabException;
+import com.epam.datalab.exceptions.InitializationException;
+import com.epam.datalab.exceptions.ParseException;
+import com.epam.datalab.util.ServiceUtils;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+
+/**
+ * Provides billing parser features.
+ */
+public class BillingTool {
+ private static final Logger LOGGER = LoggerFactory.getLogger(BillingTool.class);
+
+ /**
+ * Runs parser for given configuration.
+ *
+ * @param conf billing configuration.
+ * @throws InitializationException
+ * @throws AdapterException
+ * @throws ParseException
+ */
+ public void run(BillingToolConfiguration conf) throws InitializationException, AdapterException, ParseException {
+ ParserBase parser = conf.build();
+ LOGGER.debug("Billing Tool Configuration: {}", conf);
+ LOGGER.debug("Parser configuration: {}", parser);
+
+ parser.parse();
+ LOGGER.debug("Billing Tool statistics: {}", parser.getStatistics());
+ }
+
+ /**
+ * Runs parser for given configuration in file.
+ *
+ * @param filename the name of file for billing configuration.
+ * @throws InitializationException
+ * @throws AdapterException
+ * @throws ParseException
+ */
+ public void run(String filename) throws InitializationException, AdapterException, ParseException {
+ run(BillingToolConfigurationFactory.build(filename, BillingToolConfiguration.class));
+ }
+
+ /**
+ * Runs parser for given configuration.
+ *
+ * @param jsonNode the billing configuration.
+ * @throws InitializationException
+ * @throws AdapterException
+ * @throws ParseException
+ */
+ public void run(JsonNode jsonNode) throws InitializationException, AdapterException, ParseException {
+ run(BillingToolConfigurationFactory.build(jsonNode, BillingToolConfiguration.class));
+ }
+
+
+ /**
+ * Check the key name for command line.
+ *
+ * @param keyName the name of key.
+ * @param arg the argument from command line.
+ * @return <b>true</b> if given argument is key.
+ */
+ protected static boolean isKey(String keyName, String arg) {
+ return (("--" + keyName).equalsIgnoreCase(arg) ||
+ ("/" + keyName).equalsIgnoreCase(arg));
+ }
+
+ /**
+ * Set the level of loggers to INFO for external loggers.
+ */
+ protected static void setLoggerLevel() {
+ ch.qos.logback.classic.LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
+ ch.qos.logback.classic.Logger logger;
+ String[] loggers = {
+ "org.hibernate",
+ "org.jboss.logging"
+ };
+ for (String name : loggers) {
+ logger = context.getLogger(name);
+ logger.setLevel(Level.INFO);
+ }
+ }
+
+
+ /**
+ * Runs parser for given configuration.
+ *
+ * @param args the arguments of command line.
+ * @throws InitializationException
+ */
+ public static void main(String[] args) throws InitializationException {
+ if (ServiceUtils.printAppVersion(BillingServiceImpl.class, args)) {
+ return;
+ }
+
+ String confName = null;
+ String json = null;
+
+ for (int i = 0; i < args.length; i++) {
+ if (isKey("help", args[i])) {
+ i++;
+ Help.usage(i < args.length ? Arrays.copyOfRange(args, i, args.length) : null);
+ return;
+ } else if (isKey("conf", args[i])) {
+ i++;
+ if (i < args.length) {
+ confName = args[i];
+ } else {
+ throw new InitializationException("Missing the name of configuration file");
+ }
+ } else if (isKey("json", args[i])) {
+ i++;
+ if (i < args.length) {
+ json = args[i];
+ } else {
+ throw new InitializationException("Missing the content of json configuration");
+ }
+ } else {
+ throw new InitializationException("Unknow argument: " + args[i]);
+ }
+ }
+
+ if (confName == null && json == null) {
+ Help.usage();
+ throw new InitializationException("Missing arguments");
+ }
+
+ if (confName != null && json != null) {
+ Help.usage();
+ throw new InitializationException("Invalid arguments.");
+ }
+
+ setLoggerLevel();
+ try {
+ if (confName != null) {
+ new BillingTool().run(confName);
+ } else {
+ JsonNode jsonNode = new ObjectMapper().valueToTree(json);
+ new BillingTool().run(jsonNode);
+ }
+ } catch (Exception e) {
+ throw new DatalabException("Billing tool failed", e);
+ }
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/Help.java b/services/billing-aws/src/main/java/com/epam/datalab/Help.java
new file mode 100644
index 0000000..08096e6
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/Help.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab;
+
+import com.epam.datalab.core.BillingUtils;
+import com.epam.datalab.core.ModuleType;
+import com.epam.datalab.exceptions.InitializationException;
+import com.fasterxml.jackson.annotation.JsonClassDescription;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Print help for billing tool.
+ */
+public class Help {
+
+ private Help() {
+ }
+
+ /**
+ * Print help to console.
+ *
+ * @param resourceName the name of resource.
+ * @param substitute - map for substitution in help content.
+ * @throws InitializationException
+ */
+ private static void printHelp(String resourceName, Map<String, String> substitute) throws InitializationException {
+ List<String> list = BillingUtils.getResourceAsList("/" + Help.class.getName() + "." + resourceName + ".txt");
+ String help = StringUtils.join(list, System.lineSeparator());
+
+ if (substitute == null) {
+ substitute = new HashMap<>();
+ }
+ substitute.put("classname", BillingServiceImpl.class.getName());
+
+ for (String key : substitute.keySet()) {
+ help = StringUtils.replace(help, "${" + key.toUpperCase() + "}", substitute.get(key));
+ }
+ System.out.println(help);
+ }
+
+ /**
+ * Create and return substitutions for names of modules.
+ *
+ * @return
+ * @throws InitializationException
+ */
+ private static Map<String, String> findModules() throws InitializationException {
+ List<Class<?>> modules = BillingUtils.getModuleClassList();
+ Map<String, String> substitute = new HashMap<>();
+
+ for (Class<?> module : modules) {
+ ModuleType type = BillingUtils.getModuleType(module);
+ JsonTypeName typeName = module.getAnnotation(JsonTypeName.class);
+ if (typeName != null) {
+ String typeNames = substitute.get(type.toString() + "s");
+ typeNames = (typeNames == null ? typeName.value() : typeNames + ", " + typeName.value());
+ substitute.put(type.toString() + "s", typeNames);
+ }
+ }
+
+ return substitute;
+ }
+
+ /**
+ * Find and return help for module.
+ *
+ * @param type the type of module.
+ * @param name the name of module.
+ * @throws InitializationException
+ */
+ private static String findModuleHelp(ModuleType type, String name) throws InitializationException {
+ List<Class<?>> modules = BillingUtils.getModuleClassList();
+ String typeNames = null;
+ for (Class<?> module : modules) {
+ ModuleType t = BillingUtils.getModuleType(module);
+ if (t == type) {
+ JsonTypeName typeName = module.getAnnotation(JsonTypeName.class);
+ if (typeName != null) {
+ if (name.equals(typeName.value())) {
+ JsonClassDescription description = module.getAnnotation(JsonClassDescription.class);
+ if (description != null) {
+ return description.value();
+ }
+ throw new InitializationException("Help for " + type + " " + name + " not found");
+ } else {
+ typeNames = (typeNames == null ? typeName.value() : typeNames + ", " + typeName.value());
+ }
+ }
+ }
+ }
+ throw new InitializationException("Module for " + type + " " + name + " not found." +
+ (typeNames == null ? "" : " Module type must be one of next: " + typeNames));
+ }
+
+ /**
+ * Print help screen for billing tool.
+ *
+ * @throws InitializationException
+ */
+ public static void usage(String... args) throws InitializationException {
+ if (args == null || args.length == 0) {
+ printHelp("usage", null);
+ } else if ("conf".equalsIgnoreCase(args[0])) {
+ printHelp("conf", findModules());
+ } else {
+ ModuleType type = ModuleType.of(args[0]);
+ if (type == null) {
+ System.out.println("Unknown --help " + args[0] + " command.");
+ } else if (args.length < 2) {
+ System.out.println("Missing the type of module.");
+ String typeNames = findModules().get(type.toString() + "s");
+ if (typeNames != null) {
+ System.out.println("Must be one of next: " + typeNames);
+ }
+ } else if (args.length > 2) {
+ System.out.println("Extra arguments in command: " +
+ StringUtils.join(Arrays.copyOfRange(args, 2, args.length), " "));
+ } else {
+ System.out.println(findModuleHelp(type, args[1]));
+ }
+ }
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/conf/SecurityConfig.java b/services/billing-aws/src/main/java/com/epam/datalab/conf/SecurityConfig.java
new file mode 100644
index 0000000..d08d5fa
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/conf/SecurityConfig.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.conf;
+
+import org.keycloak.adapters.KeycloakConfigResolver;
+import org.keycloak.adapters.springboot.KeycloakSpringBootConfigResolver;
+import org.keycloak.adapters.springsecurity.KeycloakConfiguration;
+import org.keycloak.adapters.springsecurity.authentication.KeycloakAuthenticationProvider;
+import org.keycloak.adapters.springsecurity.config.KeycloakWebSecurityConfigurerAdapter;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.core.authority.mapping.SimpleAuthorityMapper;
+import org.springframework.security.core.session.SessionRegistryImpl;
+import org.springframework.security.web.authentication.session.RegisterSessionAuthenticationStrategy;
+import org.springframework.security.web.authentication.session.SessionAuthenticationStrategy;
+
+@KeycloakConfiguration
+class SecurityConfig extends KeycloakWebSecurityConfigurerAdapter {
+
+ @Autowired
+ public void configureGlobal(AuthenticationManagerBuilder auth) {
+ KeycloakAuthenticationProvider keycloakAuthenticationProvider = keycloakAuthenticationProvider();
+ keycloakAuthenticationProvider.setGrantedAuthoritiesMapper(new SimpleAuthorityMapper());
+ auth.authenticationProvider(keycloakAuthenticationProvider);
+ }
+
+ @Bean
+ public KeycloakConfigResolver keycloakConfigResolver() {
+ return new KeycloakSpringBootConfigResolver();
+ }
+
+ @Bean
+ @Override
+ protected SessionAuthenticationStrategy sessionAuthenticationStrategy() {
+ return new RegisterSessionAuthenticationStrategy(new SessionRegistryImpl());
+ }
+
+ @Override
+ protected void configure(HttpSecurity http) throws Exception {
+ super.configure(http);
+ http
+ .anonymous().disable()
+ .authorizeRequests()
+ .anyRequest()
+ .authenticated();
+ }
+}
\ No newline at end of file
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/configuration/BillingToolConfiguration.java b/services/billing-aws/src/main/java/com/epam/datalab/configuration/BillingToolConfiguration.java
new file mode 100644
index 0000000..b43031d
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/configuration/BillingToolConfiguration.java
@@ -0,0 +1,282 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.configuration;
+
+import com.epam.datalab.BillingTool;
+import com.epam.datalab.core.AdapterBase;
+import com.epam.datalab.core.AdapterBase.Mode;
+import com.epam.datalab.core.FilterBase;
+import com.epam.datalab.core.ModuleBase;
+import com.epam.datalab.core.ModuleData;
+import com.epam.datalab.core.parser.ParserBase;
+import com.epam.datalab.exceptions.AdapterException;
+import com.epam.datalab.exceptions.InitializationException;
+import com.epam.datalab.mongo.MongoDbConnection;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.MoreObjects.ToStringHelper;
+import com.google.common.collect.ImmutableList;
+
+import javax.validation.Valid;
+import javax.validation.constraints.NotNull;
+
+/**
+ * Describe configuration for {@link BillingTool}
+ */
+public class BillingToolConfiguration {
+
+ /**
+ * The host name.
+ */
+ @JsonProperty
+ private String host;
+
+ /**
+ * The port.
+ */
+ @JsonProperty
+ private int port;
+
+ /**
+ * The name of database.
+ */
+ @JsonProperty
+ private String database;
+
+ /**
+ * The name of user.
+ */
+ @JsonProperty
+ private String username;
+
+ /**
+ * The password.
+ */
+ @JsonProperty
+ private String password;
+
+ @JsonProperty
+ private boolean billingEnabled;
+
+ /**
+ * Adapter for reading source data.
+ */
+ @Valid
+ @NotNull
+ @JsonProperty
+ private ImmutableList<AdapterBase> adapterIn;
+
+ /**
+ * Adapter for writing converted data.
+ */
+ @Valid
+ @NotNull
+ @JsonProperty
+ private ImmutableList<AdapterBase> adapterOut;
+
+ /**
+ * Parser of source data to common format.
+ */
+ @Valid
+ @NotNull
+ @JsonProperty
+ private ImmutableList<ParserBase> parser;
+
+ /**
+ * Filter for source and converted data.
+ */
+ @Valid
+ @JsonProperty
+ private ImmutableList<FilterBase> filter = null;
+
+ /**
+ * Logging configuration.
+ */
+ @Valid
+ @JsonProperty
+ private LoggingConfigurationFactory logging = null;
+
+
+ /**
+ * Working data of modules.
+ */
+ @JsonIgnore
+ private ModuleData moduleData;
+
+ /**
+ * Return the adapter for reading source data.
+ */
+ public ImmutableList<AdapterBase> getAdapterIn() {
+ return adapterIn;
+ }
+
+ /**
+ * Set the adapter for reading source data.
+ */
+ public void setAdapterIn(ImmutableList<AdapterBase> adapter) {
+ for (AdapterBase a : adapter) {
+ a.setMode(Mode.READ);
+ }
+ this.adapterIn = adapter;
+ }
+
+ /**
+ * Return the adapter for writing converted data.
+ */
+ public ImmutableList<AdapterBase> getAdapterOut() {
+ return adapterOut;
+ }
+
+ /**
+ * Set the adapter for writing converted data.
+ */
+ public void setAdapterOut(ImmutableList<AdapterBase> adapter) {
+ for (AdapterBase a : adapter) {
+ a.setMode(Mode.WRITE);
+ }
+ this.adapterOut = adapter;
+ }
+
+ /**
+ * Return the parser of source data to common format.
+ */
+ public ImmutableList<ParserBase> getParser() {
+ return parser;
+ }
+
+ /**
+ * Set the parser of source data to common format.
+ */
+ public void setParser(ImmutableList<ParserBase> parser) {
+ this.parser = parser;
+ }
+
+ /**
+ * Return the filter for source and converted data.
+ */
+ public ImmutableList<FilterBase> getFilter() {
+ return filter;
+ }
+
+ /**
+ * Set the filter for source and converted data.
+ */
+ public void setFilter(ImmutableList<FilterBase> filter) {
+ this.filter = filter;
+ }
+
+ /**
+ * Return the logging configuration.
+ */
+ public LoggingConfigurationFactory getLogging() {
+ return logging;
+ }
+
+ /**
+ * Set the logging configuration.
+ */
+ public void setLogging(LoggingConfigurationFactory logging) {
+ this.logging = logging;
+ }
+
+
+ /**
+ * Return the working data of modules.
+ */
+ @JsonIgnore
+ public ModuleData getModuleData() {
+ return moduleData;
+ }
+
+ /**
+ * Check and return module.
+ *
+ * @param modules the list of modules.
+ * @param name the name of module.
+ * @param isOptional optional module or not.
+ * @return module
+ * @throws InitializationException
+ */
+ private <T extends ModuleBase> T getModule(ImmutableList<T> modules, String name, boolean isOptional) throws
+ InitializationException {
+ T module = (modules != null && modules.size() == 1 ? modules.get(0) : null);
+ if (!isOptional && module == null) {
+ throw new InitializationException("Invalid configuration for property " + name);
+ }
+ return module;
+ }
+
+ /**
+ * Build and return the parser.
+ *
+ * @return the parser.
+ * @throws InitializationException
+ */
+ public ParserBase build() throws InitializationException {
+ ParserBase parserBase = getModule(this.parser, "parser", false);
+ AdapterBase in = getModule(adapterIn, "adapterIn", false);
+ AdapterBase out = getModule(adapterOut, "adapterOut", false);
+ FilterBase f = getModule(filter, "filter", true);
+
+ final MongoDbConnection connection;
+ try {
+ connection = new MongoDbConnection(host, port, database, username, password);
+ } catch (AdapterException e) {
+ throw new InitializationException("Cannot configure mongo connection. " + e.getLocalizedMessage(), e);
+ }
+ moduleData = new ModuleData(connection);
+
+ parserBase.setModuleData(moduleData);
+ in.setModuleData(moduleData);
+ out.setModuleData(moduleData);
+ if (f != null) {
+ f.setModuleData(moduleData);
+ }
+
+ return parserBase.build(in, out, f);
+ }
+
+ public boolean isBillingEnabled() {
+ return billingEnabled;
+ }
+
+ /**
+ * Returns a string representation of the object.
+ *
+ * @param self the object to generate the string for (typically this), used only for its class name.
+ */
+ public ToStringHelper toStringHelper(Object self) {
+ return MoreObjects.toStringHelper(self)
+ .add("moduleData", moduleData)
+ .add("adapterIn", adapterIn)
+ .add("adapterOut", adapterOut)
+ .add("filter", filter)
+ .add("parser", parser)
+ .add("logging", logging)
+ .add("billingEnabled", billingEnabled);
+ }
+
+ @Override
+ public String toString() {
+ return toStringHelper(this)
+ .toString();
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/configuration/BillingToolConfigurationFactory.java b/services/billing-aws/src/main/java/com/epam/datalab/configuration/BillingToolConfigurationFactory.java
new file mode 100644
index 0000000..fac6795
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/configuration/BillingToolConfigurationFactory.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.configuration;
+
+import com.epam.datalab.core.BillingUtils;
+import com.epam.datalab.exceptions.InitializationException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.fasterxml.jackson.datatype.guava.GuavaModule;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Build the instance of class {@link BillingToolConfiguration}.
+ */
+public class BillingToolConfigurationFactory {
+
+ /**
+ * Mapper for reading configuration.
+ */
+ private static ObjectMapper mapper;
+
+ /**
+ * Build the instance of class {@link BillingToolConfiguration} from YAML file.
+ *
+ * @param filename the name of file.
+ * @param confClass configuration class.
+ * @return the instance of configuration.
+ * @throws InitializationException
+ */
+ public static <T extends BillingToolConfiguration> T build(String filename, Class<T> confClass) throws InitializationException {
+ try {
+ InputStream is = new FreeMarkerConfig().getInputStream(filename);
+ JsonNode node = getMapper().readTree(new YAMLFactory().createParser(is));
+ return build(node, confClass);
+ } catch (IOException | InitializationException e) {
+ throw new InitializationException("Cannot parse configuration file " + filename + ". " + e.getLocalizedMessage(), e);
+ }
+ }
+
+ /**
+ * Build the instance of class {@link BillingToolConfiguration} from YAML file.
+ *
+ * @param node the content of configuration.
+ * @param confClass configuration class.
+ * @return the instance of configuration.
+ * @throws InitializationException
+ */
+ public static <T extends BillingToolConfiguration> T build(JsonNode node, Class<T> confClass) throws InitializationException {
+ T conf;
+ try {
+ conf = getMapper().readValue(node.toString(), confClass);
+ } catch (Exception e) {
+ throw new InitializationException("Cannot parse json configuration. " + e.getLocalizedMessage(), e);
+ }
+
+ try {
+ LoggingConfigurationFactory logging = conf.getLogging();
+ if (logging != null) {
+ logging.configure();
+ }
+ } catch (Exception e) {
+ throw new InitializationException("Cannot initialize configuration. " + e.getLocalizedMessage(), e);
+ }
+
+ new ConfigurationValidator<T>()
+ .validate(conf);
+
+ return conf;
+ }
+
+ /**
+ * Return the mapper for reading configuration.
+ *
+ * @throws InitializationException
+ */
+ private static ObjectMapper getMapper() throws InitializationException {
+ if (mapper != null) {
+ return mapper;
+ }
+ mapper = new ObjectMapper()
+ .enable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+ mapper.registerModule(new GuavaModule());
+ for (Class<?> clazz : BillingUtils.getModuleClassList()) {
+ mapper.registerSubtypes(clazz);
+ }
+
+ return mapper;
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/configuration/ConfigJsonGenerator.java b/services/billing-aws/src/main/java/com/epam/datalab/configuration/ConfigJsonGenerator.java
new file mode 100644
index 0000000..fae1bf6
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/configuration/ConfigJsonGenerator.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.configuration;
+
+import com.epam.datalab.core.BillingUtils;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Generate the json configuration of billing tool.
+ */
+public class ConfigJsonGenerator {
+
+ /**
+ * Buffer for configuration properties.
+ */
+ private Map<String, Map<String, String>[]> config = new HashMap<>();
+
+ /**
+ * Add the properties of module to configuration.
+ *
+ * @param moduleName the name of module.
+ * @param properties the properties: key and value sequence.
+ */
+ private ConfigJsonGenerator withModule(String moduleName, String... properties) {
+ if (properties == null) {
+ config.remove(moduleName);
+ return this;
+ }
+
+ @SuppressWarnings("unchecked")
+ Map<String, String>[] map = new Map[1];
+ map[0] = BillingUtils.stringsToMap(properties);
+ config.put(moduleName, map);
+ return this;
+ }
+
+ /**
+ * Add the properties of input adapter to configuration.
+ *
+ * @param properties the properties: key and value sequence.
+ */
+ public ConfigJsonGenerator withAdapterIn(String... properties) {
+ return withModule("adapterIn", properties);
+ }
+
+ /**
+ * Add the properties of output adapter to configuration.
+ *
+ * @param properties the properties: key and value sequence.
+ */
+ public ConfigJsonGenerator withAdapterOut(String... properties) {
+ return withModule("adapterOut", properties);
+ }
+
+ /**
+ * Add the properties of parser to configuration.
+ *
+ * @param properties the properties: key and value sequence.
+ */
+ public ConfigJsonGenerator withParser(String... properties) {
+ return withModule("parser", properties);
+ }
+
+ /**
+ * Add the properties of filter to configuration.
+ *
+ * @param properties the properties: key and value sequence.
+ */
+ public ConfigJsonGenerator withFilter(String... properties) {
+ return withModule("filter", properties);
+ }
+
+ /**
+ * Build and return json configuration.
+ *
+ * @param properties the properties: key and value sequence.
+ */
+ public JsonNode build() {
+ return new ObjectMapper()
+ .valueToTree(config);
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/configuration/ConfigurationValidator.java b/services/billing-aws/src/main/java/com/epam/datalab/configuration/ConfigurationValidator.java
new file mode 100644
index 0000000..4f8c039
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/configuration/ConfigurationValidator.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.configuration;
+
+import com.epam.datalab.core.BillingUtils;
+import com.epam.datalab.exceptions.InitializationException;
+
+import javax.validation.ConstraintViolation;
+import javax.validation.Validation;
+import javax.validation.Validator;
+import javax.validation.ValidatorFactory;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Json properties validator.
+ *
+ * @param <T> Class for validation.
+ */
+public class ConfigurationValidator<T> {
+
+ /**
+ * Error messages.
+ */
+ private static Map<String, String> messages = BillingUtils.stringsToMap(
+ "{javax.validation.constraints.NotNull.message}", "Property \"%s\" may not be null");
+
+ /**
+ * Return the list of error messages.
+ *
+ * @param violation constraint violations.
+ */
+ public String getMessage(ConstraintViolation<T> violation) {
+ return String.format(
+ messages.get(violation.getMessageTemplate()),
+ violation.getPropertyPath(),
+ violation.getInvalidValue());
+ }
+
+ /**
+ * Validate properties in instance and throw exception if it have not valid property.
+ *
+ * @param clazz instance for validation.
+ * @throws InitializationException
+ */
+ public void validate(T clazz) throws InitializationException {
+ ValidatorFactory validatorFactory = Validation.buildDefaultValidatorFactory();
+ Validator validator = validatorFactory.getValidator();
+ Set<ConstraintViolation<T>> violations = validator.validate(clazz);
+ for (ConstraintViolation<T> violation : violations) {
+ throw new InitializationException(getMessage(violation));
+ }
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/configuration/FreeMarkerConfig.java b/services/billing-aws/src/main/java/com/epam/datalab/configuration/FreeMarkerConfig.java
new file mode 100644
index 0000000..a134f30
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/configuration/FreeMarkerConfig.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.configuration;
+
+import com.google.common.base.Throwables;
+import freemarker.template.Configuration;
+import freemarker.template.Template;
+import freemarker.template.TemplateException;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * Provides Apache FreeMarker the template engine for the billing configuration.
+ */
+public class FreeMarkerConfig {
+
+ /**
+ * Create and return the input stream for the configuration file.
+ *
+ * @param filename the name of configuration file.
+ * @throws IOException
+ */
+ public InputStream getInputStream(final String filename) throws IOException {
+ try {
+ Configuration conf = new Configuration(Configuration.VERSION_2_3_22);
+ Template template = new Template("billing-config", new FileReader(new File(filename)), conf);
+ Map<String, Object> dataModel = getDataModel();
+
+ ByteArrayOutputStream streamBuffer = new ByteArrayOutputStream();
+ template.process(dataModel, new OutputStreamWriter(streamBuffer, StandardCharsets.UTF_8));
+ byte[] buffer = streamBuffer.toByteArray();
+
+ return new ByteArrayInputStream(buffer);
+ } catch (TemplateException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ /**
+ * Create and return JVM and OS properties.
+ */
+ private Map<String, Object> getDataModel() {
+ Map<String, Object> dataModel = new HashMap<>();
+
+ Iterator<Object> sysProps = System.getProperties().keySet().iterator();
+ while (sysProps.hasNext()) {
+ String key = (String) sysProps.next();
+ dataModel.put(key, System.getProperties().getProperty(key));
+ }
+ dataModel.putAll(System.getenv());
+
+ dataModel.put("env", System.getenv());
+ dataModel.put("sys", System.getProperties());
+
+ return dataModel;
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/configuration/LoggingConfigurationFactory.java b/services/billing-aws/src/main/java/com/epam/datalab/configuration/LoggingConfigurationFactory.java
new file mode 100644
index 0000000..e407c14
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/configuration/LoggingConfigurationFactory.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.configuration;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import com.epam.datalab.exceptions.InitializationException;
+import com.epam.datalab.logging.AppenderBase;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.MoreObjects.ToStringHelper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.slf4j.LoggerFactory;
+
+import javax.validation.Valid;
+
+/**
+ * Configuration and factory for logging.
+ */
+public class LoggingConfigurationFactory {
+
+ /**
+ * Default logging level for all appenders.
+ */
+ @Valid
+ @JsonProperty
+ private Level level = Level.INFO;
+
+ /**
+ * List of logging levels for appenders.
+ */
+ @JsonIgnore
+ private ImmutableMap<String, Level> loggers = ImmutableMap.of();
+
+ /**
+ * List of logging appenders.
+ */
+ @Valid
+ @JsonProperty
+ private ImmutableList<AppenderBase> appenders = ImmutableList.of();
+
+
+ /**
+ * Return the default logging level for all appenders.
+ */
+ public Level getLevel() {
+ return level;
+ }
+
+ /**
+ * Set the default logging level for all appenders.
+ */
+ public void setLevel(String level) throws InitializationException {
+ this.level = toLevel(level);
+ }
+
+ /**
+ * Return the list of logging levels for appenders.
+ */
+ public ImmutableMap<String, Level> getLoggers() {
+ return loggers;
+ }
+
+ /**
+ * Set the list of logging levels for appenders.
+ */
+ @JsonProperty
+ public void setLoggers(ImmutableMap<String, JsonNode> loggers) throws InitializationException {
+ ImmutableMap.Builder<String, Level> levels = new ImmutableMap.Builder<>();
+ for (String key : loggers.keySet()) {
+ JsonNode node = loggers.get(key);
+ levels.put(key, toLevel(node.asText()));
+ }
+ this.loggers = levels.build();
+ }
+
+ /**
+ * Return the list of logging appenders.
+ */
+ public ImmutableList<AppenderBase> getAppenders() {
+ return appenders;
+ }
+
+ /**
+ * Set the list of logging appenders.
+ */
+ public void setAppenders(ImmutableList<AppenderBase> appenders) {
+ this.appenders = appenders;
+ }
+
+
+ /**
+ * Translate the name of logging level to {@link Level}.
+ *
+ * @param level the name of logging level.
+ * @return logging level.
+ * @throws InitializationException if given unknown logging level name.
+ */
+ private Level toLevel(String level) throws InitializationException {
+ Level l = Level.toLevel(level, null);
+ if (l == null) {
+ throw new InitializationException("Unknown logging level: " + level);
+ }
+ return l;
+ }
+
+ /**
+ * Configure logging appenders.
+ *
+ * @throws InitializationException
+ */
+ public void configure() throws InitializationException {
+ if (appenders == null) {
+ throw new InitializationException("Configuration property logging.appenders cannot be null.");
+ }
+ LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
+ context.reset();
+
+ for (AppenderBase appender : appenders) {
+ appender.configure(context);
+ }
+
+ Logger logger = context.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME);
+ logger.setLevel(level);
+ for (String name : loggers.keySet()) {
+ logger = context.getLogger(name);
+ logger.setLevel(loggers.get(name));
+ }
+ }
+
+
+ /**
+ * Returns a string representation of the object.
+ *
+ * @param self the object to generate the string for (typically this), used only for its class name.
+ */
+ public ToStringHelper toStringHelper(Object self) {
+ return MoreObjects.toStringHelper(self)
+ .add("level", level)
+ .add("loggers", loggers)
+ .add("appenders", appenders);
+ }
+
+ @Override
+ public String toString() {
+ return toStringHelper(this)
+ .toString();
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/controller/BillingController.java b/services/billing-aws/src/main/java/com/epam/datalab/controller/BillingController.java
new file mode 100644
index 0000000..d28d9f4
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/controller/BillingController.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.controller;
+
+import com.epam.datalab.BillingService;
+import com.epam.datalab.dto.billing.BillingData;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+
+@RestController
+public class BillingController {
+
+ private final BillingService billingService;
+
+ public BillingController(BillingService billingService) {
+ this.billingService = billingService;
+ }
+
+ @GetMapping
+ public ResponseEntity<List<BillingData>> getBilling() {
+ return new ResponseEntity<>(billingService.getBillingData(), HttpStatus.OK);
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/AdapterBase.java b/services/billing-aws/src/main/java/com/epam/datalab/core/AdapterBase.java
new file mode 100644
index 0000000..9ceea05
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/AdapterBase.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core;
+
+import com.epam.datalab.exceptions.AdapterException;
+import com.epam.datalab.model.aws.ReportLine;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.MoreObjects.ToStringHelper;
+import org.bson.Document;
+
+import java.util.List;
+
+/**
+ * Abstract module for read/write adapter.
+ * See description of {@link ModuleBase} how to create your own adapter.
+ */
+public abstract class AdapterBase extends ModuleBase {
+ public enum Mode {READ, WRITE}
+
+ ;
+
+ /**
+ * Flag the header of common format should be written to target.
+ */
+ @JsonProperty
+ private boolean writeHeader = true;
+
+
+ /**
+ * The mode of adapter read or write.
+ */
+ @JsonIgnore
+ private Mode mode;
+
+
+ /**
+ * Default constructor for deserialization.
+ */
+ public AdapterBase() {
+ }
+
+ /**
+ * Instantiate adapter for reading or writing.
+ *
+ * @param mode the mode of adapter.
+ */
+ public AdapterBase(Mode mode) {
+ this.mode = mode;
+ }
+
+
+ /**
+ * Return <b>true</b> if the header of common format should be written to target.
+ */
+ public boolean isWriteHeader() {
+ return writeHeader;
+ }
+
+ /**
+ * Set flag the header of common format should be written to target.
+ */
+ public void setWriteHeader(boolean writeHeader) {
+ this.writeHeader = writeHeader;
+ }
+
+
+ /**
+ * Return the mode of adapter read or write.
+ */
+ public Mode getMode() {
+ return mode;
+ }
+
+ /**
+ * Set the mode of adapter read or write.
+ */
+ public void setMode(Mode mode) {
+ this.mode = mode;
+ }
+
+
+ /**
+ * Open connection.
+ *
+ * @throws AdapterException if cannot open connection.
+ */
+ public abstract void open() throws AdapterException;
+
+ /**
+ * Return <b>true</b> if adapter has the multiply entries of data.
+ */
+ public boolean hasMultyEntry() {
+ return false;
+ }
+
+ /**
+ * Return <b>true</b> if current entry has the data.
+ */
+ public boolean hasEntryData() {
+ return true;
+ }
+
+ /**
+ * Open next entry if exists and return <b>true</b> otherwise return <b>false</b>.
+ *
+ * @throws AdapterException if cannot open entry.
+ */
+ public boolean openNextEntry() throws AdapterException {
+ return false;
+ }
+
+ /**
+ * Close connection.
+ *
+ * @throws AdapterException if cannot close connection.
+ */
+ public abstract void close() throws AdapterException;
+
+ /**
+ * Return the current processed entry name.
+ */
+ public abstract String getEntryName();
+
+ /**
+ * Read the line of data from adapter and return it.
+ *
+ * @throws AdapterException
+ */
+ public abstract String readLine() throws AdapterException;
+
+ /**
+ * Write the header of data to adapter.
+ *
+ * @param header the header of common format.
+ * @throws AdapterException
+ */
+ public abstract void writeHeader(List<String> header) throws AdapterException;
+
+ /**
+ * Write the row of data to adapter.
+ *
+ * @param row the row of common format.
+ * @return
+ * @throws AdapterException
+ */
+ public abstract Document writeRow(ReportLine row) throws AdapterException;
+
+
+ @Override
+ public ToStringHelper toStringHelper(Object self) {
+ return super.toStringHelper(self)
+ .add("mode", mode)
+ .add("writeHeader", isWriteHeader());
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/BillingUtils.java b/services/billing-aws/src/main/java/com/epam/datalab/core/BillingUtils.java
new file mode 100644
index 0000000..65ec1b5
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/BillingUtils.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core;
+
+import com.epam.datalab.configuration.BillingToolConfigurationFactory;
+import com.epam.datalab.core.parser.ParserBase;
+import com.epam.datalab.exceptions.InitializationException;
+import com.epam.datalab.logging.AppenderBase;
+import com.google.common.io.Resources;
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Billing toll utilities.
+ */
+public class BillingUtils {
+
+ /**
+ * Name of resource with the names of module classes.
+ */
+ private static final String RESOURCE_MODULE_NAMES = "/" + BillingToolConfigurationFactory.class.getName();
+
+ private BillingUtils() {
+ }
+
+ /**
+ * Create and return map from given key/values.
+ *
+ * @param keyValues the key/value pairs.
+ */
+ public static Map<String, String> stringsToMap(String... keyValues) {
+ Map<String, String> map = new HashMap<>();
+ if (keyValues == null) {
+ return map;
+ }
+ if (keyValues.length % 2 != 0) {
+ throw new IllegalArgumentException("Missing key or value in arguments");
+ }
+
+ for (int i = 1; i < keyValues.length; i += 2) {
+ map.put(keyValues[i - 1], keyValues[i]);
+ }
+ return map;
+ }
+
+ /**
+ * Read and return content as string list from resource.
+ *
+ * @param resourceName the name of resource.
+ * @return list of strings.
+ * @throws InitializationException
+ */
+ public static List<String> getResourceAsList(String resourceName) throws InitializationException {
+ try {
+ URL url = BillingToolConfigurationFactory.class.getResource(resourceName);
+ if (url == null) {
+ throw new InitializationException("Resource " + resourceName + " not found");
+ }
+ return Resources.readLines(url, Charset.forName("utf-8"));
+ } catch (IllegalArgumentException | IOException e) {
+ throw new InitializationException("Cannot read resource " + resourceName + ": " + e.getLocalizedMessage(), e);
+ }
+ }
+
+ /**
+ * Return the list of billing tool modules.
+ *
+ * @throws InitializationException
+ */
+ public static List<Class<?>> getModuleClassList() throws InitializationException {
+ List<String> modules = getResourceAsList(RESOURCE_MODULE_NAMES);
+ List<Class<?>> classes = new ArrayList<>();
+
+ for (String className : modules) {
+ try {
+ classes.add(Class.forName(className));
+ } catch (ClassNotFoundException e) {
+ throw new InitializationException("Cannot add the sub type " + className +
+ " from resource " + RESOURCE_MODULE_NAMES + ": " + e.getLocalizedMessage(), e);
+ }
+ }
+ return classes;
+ }
+
+ /**
+ * Check for child class is belong to parent by hierarchy.
+ *
+ * @param child the child class for check.
+ * @param parent the parent class from hierarchy.
+ */
+ public static boolean classChildOf(Class<?> child, Class<?> parent) {
+ return child != null && parent != null && parent.isAssignableFrom(child);
+ }
+
+ /**
+ * Return the type of module if class is module otherwise <b>null</b>.
+ *
+ * @param moduleClass the class.
+ */
+ public static ModuleType getModuleType(Class<?> moduleClass) {
+ if (classChildOf(moduleClass, AdapterBase.class)) {
+ return ModuleType.ADAPTER;
+ } else if (classChildOf(moduleClass, FilterBase.class)) {
+ return ModuleType.FILTER;
+ } else if (classChildOf(moduleClass, ParserBase.class)) {
+ return ModuleType.PARSER;
+ } else if (classChildOf(moduleClass, AppenderBase.class)) {
+ return ModuleType.LOGAPPENDER;
+ }
+ return null;
+ }
+
+ /**
+ * Return the name of user without domain.
+ *
+ * @param value the value.
+ */
+ public static String getSimpleUserName(String username) {
+ return (username == null ? null : username.replaceAll("@.*", ""));
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/DBAdapterBase.java b/services/billing-aws/src/main/java/com/epam/datalab/core/DBAdapterBase.java
new file mode 100644
index 0000000..cda3da0
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/DBAdapterBase.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.MoreObjects.ToStringHelper;
+
+/**
+ * The abstract adapter for database.
+ * See description of {@link ModuleBase} how to create your own adapter.
+ */
+public abstract class DBAdapterBase extends AdapterBase {
+
+ /**
+ * The host name.
+ */
+ @JsonProperty
+ private String host;
+
+ /**
+ * The port.
+ */
+ @JsonProperty
+ private int port;
+
+ /**
+ * The name of database.
+ */
+ @JsonProperty
+ private String database;
+
+ /**
+ * The name of user.
+ */
+ @JsonProperty
+ private String username;
+
+ /**
+ * The password.
+ */
+ @JsonProperty
+ private String password;
+
+
+ /**
+ * Default constructor for deserialization.
+ */
+ public DBAdapterBase() {
+ }
+
+ /**
+ * Instantiate adapter for reading or writing.
+ *
+ * @param mode the mode of adapter.
+ */
+ public DBAdapterBase(Mode mode) {
+ super(mode);
+ }
+
+
+ @Override
+ public boolean isWriteHeader() {
+ return false;
+ }
+
+ /**
+ * Return the host name.
+ */
+ public String getHost() {
+ return host;
+ }
+
+ /**
+ * Set the host name.
+ */
+ public void setHost(String host) {
+ this.host = host;
+ }
+
+ /**
+ * Return the port.
+ */
+ public int getPort() {
+ return port;
+ }
+
+ /**
+ * Set the port.
+ */
+ public void setPort(int port) {
+ this.port = port;
+ }
+
+ /**
+ * Return the name of database.
+ */
+ public String getDatabase() {
+ return database;
+ }
+
+ /**
+ * Set the name of database.
+ */
+ public void setDatabase(String database) {
+ this.database = database;
+ }
+
+ /**
+ * Return the name of user.
+ */
+ public String getUsername() {
+ return username;
+ }
+
+ /**
+ * Set the name of user.
+ */
+ public void setUsername(String username) {
+ this.username = username;
+ }
+
+ /**
+ * Return the password.
+ */
+ public String getPassword() {
+ return password;
+ }
+
+ /**
+ * Set the password.
+ */
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+
+ @Override
+ public ToStringHelper toStringHelper(Object self) {
+ return super.toStringHelper(self)
+ .add("host", host)
+ .add("port", port)
+ .add("database", database)
+ .add("username", username)
+ .add("password", "***");
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/FilterBase.java b/services/billing-aws/src/main/java/com/epam/datalab/core/FilterBase.java
new file mode 100644
index 0000000..4e17514
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/FilterBase.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core;
+
+import com.epam.datalab.core.parser.ParserBase;
+import com.epam.datalab.exceptions.InitializationException;
+import com.epam.datalab.exceptions.ParseException;
+import com.epam.datalab.model.aws.ReportLine;
+import com.google.common.base.MoreObjects.ToStringHelper;
+
+import java.util.List;
+
+/**
+ * Abstract module of filtering.
+ * See description of {@link ModuleBase} how to create your own filter.
+ */
+public abstract class FilterBase extends ModuleBase {
+
+ /**
+ * Parser.
+ */
+ private ParserBase parser;
+
+ /**
+ * Return parser.
+ */
+ public ParserBase getParser() {
+ return parser;
+ }
+
+ /**
+ * Set parser.
+ */
+ public void setParser(ParserBase parser) {
+ this.parser = parser;
+ }
+
+
+ /**
+ * Initialize the filter.
+ *
+ * @throws InitializationException
+ */
+ public abstract void initialize() throws InitializationException;
+
+ /**
+ * Return the line for parsing if line is accepted and may be parsed,
+ * otherwise return <b>null</b>.
+ *
+ * @param line the source line.
+ * @throws ParseException
+ */
+ public abstract String canParse(String line) throws ParseException;
+
+ /**
+ * Return the list of values for transformation if value is accepted and may be transformed,
+ * otherwise return <b>null</b>.
+ *
+ * @param row the list of values.
+ * @throws ParseException
+ */
+ public abstract List<String> canTransform(List<String> row) throws ParseException;
+
+ /**
+ * Return the row of billing report if row is accepted and may be written to target,
+ * otherwise return <b>null</b>.
+ *
+ * @param row the report line.
+ * @throws ParseException
+ */
+ public abstract ReportLine canAccept(ReportLine row) throws ParseException;
+
+ @Override
+ public ToStringHelper toStringHelper(Object self) {
+ return super.toStringHelper(self)
+ .add("parser", (parser == null ? null : parser.getType()));
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/ModuleBase.java b/services/billing-aws/src/main/java/com/epam/datalab/core/ModuleBase.java
new file mode 100644
index 0000000..acb26a3
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/ModuleBase.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core;
+
+import com.epam.datalab.core.parser.ParserBase;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.MoreObjects.ToStringHelper;
+
+/**
+ * Abstract class for modules: adapter, filter, parser.<br>
+ * To create your adapter:<br>
+ * 1. Create a class which extends one of {@link AdapterBase}, {@link FilterBase} or {@link ParserBase} classes.<br>
+ * 2. Annotate it with {@link JsonTypeName} annotation and give it a unique type name for this type of modules.<br>
+ * 3. Add full the name of your class to main/resources/com.epam.datalab.configuration.BillingToolConfigurationFactory file.
+ * 4. Annotate it with {@link JsonClassDescription] annotation and describe all properties of module.
+ */
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+public abstract class ModuleBase {
+
+ /**
+ * Working data of module.
+ */
+ @JsonIgnore
+ private ModuleData moduleData;
+
+ /**
+ * Return the name of type for appender.
+ */
+ @JsonIgnore
+ public String getType() {
+ Class<? extends ModuleBase> clazz = this.getClass();
+ return (clazz.isAnnotationPresent(JsonTypeName.class) ?
+ clazz.getAnnotation(JsonTypeName.class).value() : clazz.getName());
+ }
+
+ /**
+ * Return the working data of module.
+ */
+ public ModuleData getModuleData() {
+ return moduleData;
+ }
+
+ /**
+ * Set the working data of module.
+ */
+ public void setModuleData(ModuleData moduleData) {
+ this.moduleData = moduleData;
+ }
+
+ /**
+ * Returns a string representation of the object.
+ *
+ * @param self the object to generate the string for (typically this), used only for its class name.
+ */
+ public ToStringHelper toStringHelper(Object self) {
+ return MoreObjects.toStringHelper(self)
+ .add("type", getType());
+ }
+
+ @Override
+ public String toString() {
+ return toStringHelper(this).toString();
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/ModuleData.java b/services/billing-aws/src/main/java/com/epam/datalab/core/ModuleData.java
new file mode 100644
index 0000000..41e1bdd
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/ModuleData.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core;
+
+import com.epam.datalab.exceptions.InitializationException;
+import com.epam.datalab.mongo.MongoConstants;
+import com.epam.datalab.mongo.MongoDbConnection;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.MoreObjects.ToStringHelper;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.model.UpdateOptions;
+import org.apache.commons.lang3.StringUtils;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import static com.mongodb.client.model.Filters.and;
+import static com.mongodb.client.model.Filters.eq;
+import static com.mongodb.client.model.Filters.gte;
+import static com.mongodb.client.model.Filters.or;
+import static com.mongodb.client.model.Filters.regex;
+
+/**
+ * Provides loading and storing the working data of modules.
+ */
+public class ModuleData {
+
+ public static final String ENTRIES_FIELD = "entries";
+ private static final String ID_FIELD = "_id";
+ private static final String MODIFICATION_DATE = "lastModificationDate";
+ /**
+ * Date formatter.
+ */
+ private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
+ private final MongoDbConnection connection;
+
+ private String id;
+ private Date modificationDate;
+
+ /**
+ * Entries of data.
+ */
+ private Map<String, String> entries = new HashMap<>();
+
+ /**
+ * Flag modification of entries.
+ */
+ private boolean modified;
+
+ /**
+ * Instantiate module data.
+ *
+ * @param connection the name of data file.
+ * @throws InitializationException
+ */
+ public ModuleData(MongoDbConnection connection) {
+ this.connection = connection;
+ }
+
+ /**
+ * Return <b>true</b> if any entries was modify.
+ */
+ public boolean isModified() {
+ return modified;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public void setModificationDate(Date modificationDate) {
+ this.modificationDate = modificationDate;
+ }
+
+ /**
+ * Return the value for given key or <b>null</b> if value not found.
+ *
+ * @param key the key of entry.
+ */
+ public String getString(String key) {
+ return entries.get(key);
+ }
+
+ /**
+ * Return the date value for given key or <b>null</b> if value not found.
+ *
+ * @param key the key of entry.
+ * @throws ParseException
+ */
+ public Date getDate(String key) throws ParseException {
+ String value = entries.get(key);
+ return (value == null ? null : dateFormat.parse(value));
+ }
+
+ /**
+ * Set value for given key or delete entry if the value is <b>null</b>.
+ *
+ * @param key the key of entry.
+ * @param value the value.
+ */
+ public void set(String key, String value) {
+ if (StringUtils.equals(entries.get(key), value)) {
+ return;
+ } else if (value == null) {
+ entries.remove(key);
+ } else {
+ entries.put(key, value);
+ }
+ modified = true;
+ }
+
+ /**
+ * Set value for given key or delete entry if the value is <b>null</b>.
+ *
+ * @param key the key of entry.
+ * @param value the date.
+ */
+ public void set(String key, Date value) {
+ set(key, dateFormat.format(value));
+ }
+
+ public void store() {
+ final Document document = new Document().append(ID_FIELD, id).append(MODIFICATION_DATE, modificationDate).append(ENTRIES_FIELD, entries);
+ connection.getCollection(MongoConstants.BILLING_DATA_COLLECTION)
+ .updateOne(eq(ID_FIELD, id), new Document("$set", document), new UpdateOptions().upsert(true));
+ modified = false;
+ }
+
+ public boolean wasProcessed(String fileName, Date modificationDate, String datePrefix) {
+ final Bson filePerBillingPeriodCondition = and(regex(ID_FIELD, "^" + datePrefix), gte(MODIFICATION_DATE,
+ modificationDate));
+ final Bson fileWithExactNameCondition = and(eq(ID_FIELD, fileName), gte(MODIFICATION_DATE, modificationDate));
+ final FindIterable<Document> documents = connection.getCollection(MongoConstants.BILLING_DATA_COLLECTION)
+ .find(or(fileWithExactNameCondition, filePerBillingPeriodCondition))
+ .limit(1);
+ return documents.iterator().hasNext();
+ }
+
+ public void closeMongoConnection() throws IOException {
+ connection.close();
+ }
+
+
+ /**
+ * Returns a string representation of the object.
+ *
+ * @param self the object to generate the string for (typically this), used only for its class name.
+ */
+ public ToStringHelper toStringHelper(Object self) {
+ return MoreObjects.toStringHelper(self)
+ .addValue(entries);
+ }
+
+ @Override
+ public String toString() {
+ return toStringHelper(this)
+ .toString();
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/ModuleType.java b/services/billing-aws/src/main/java/com/epam/datalab/core/ModuleType.java
new file mode 100644
index 0000000..1b30446
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/ModuleType.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core;
+
+/**
+ * The type of billing modules: adapter, filter, parser or log appender.
+ */
+public enum ModuleType {
+ ADAPTER,
+ FILTER,
+ PARSER,
+ LOGAPPENDER;
+
+ @Override
+ public String toString() {
+ return super.toString().toLowerCase();
+ }
+
+ public static ModuleType of(String string) {
+ if (string != null) {
+ for (ModuleType value : ModuleType.values()) {
+ if (string.equalsIgnoreCase(value.toString())) {
+ return value;
+ }
+ }
+ }
+ return null;
+ }
+}
\ No newline at end of file
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/aggregate/AggregateGranularity.java b/services/billing-aws/src/main/java/com/epam/datalab/core/aggregate/AggregateGranularity.java
new file mode 100644
index 0000000..20617d2
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/aggregate/AggregateGranularity.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.epam.datalab.core.aggregate;
+
+/**
+ * Aggregate granularity for aggregation.
+ */
+public enum AggregateGranularity {
+ NONE,
+ DAY,
+ MONTH;
+
+ @Override
+ public String toString() {
+ return super.toString().toLowerCase();
+ }
+
+ public static AggregateGranularity of(String string) {
+ if (string != null) {
+ for (AggregateGranularity value : AggregateGranularity.values()) {
+ if (string.equalsIgnoreCase(value.toString())) {
+ return value;
+ }
+ }
+ }
+ return null;
+ }
+}
\ No newline at end of file
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/aggregate/DataAggregator.java b/services/billing-aws/src/main/java/com/epam/datalab/core/aggregate/DataAggregator.java
new file mode 100644
index 0000000..0800182
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/aggregate/DataAggregator.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core.aggregate;
+
+import com.epam.datalab.model.aws.ReportLine;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Vector;
+
+/**
+ * Aggregate billing report and summarizes column usage and cost.
+ */
+public class DataAggregator {
+ /**
+ * List of the report lines.
+ */
+ private final Vector<ReportLine> reportLines = new Vector<>(1000);
+
+ /**
+ * Comparator for aggregation.
+ */
+ private final Comparator<ReportLine> aggComparator = new AggComparator();
+
+ /**
+ * Granularity for aggregation.
+ */
+ private AggregateGranularity granularity;
+
+ /**
+ * Length of date for truncate.
+ */
+ private int truncateDateLength;
+
+
+ public DataAggregator(AggregateGranularity granularity) {
+ switch (granularity) {
+ case DAY:
+ truncateDateLength = 10;
+ break;
+ case MONTH:
+ truncateDateLength = 7;
+ break;
+ default:
+ throw new IllegalArgumentException("Invalid value of granularity argument: expected DAY or MONTH, " +
+ "actual is " + granularity);
+ }
+ this.granularity = granularity;
+ }
+
+ /**
+ * Return granularity for aggregation.
+ */
+ public AggregateGranularity getGranularity() {
+ return granularity;
+ }
+
+ /**
+ * Appends the report line to the list and returns it.
+ *
+ * @param row the line of report.
+ * @return Instance of the aggregated report line.
+ */
+ public ReportLine append(ReportLine row) {
+ synchronized (this) {
+ String usageInterval = truncDate(row.getUsageDate());
+ row.setUsageDate(usageInterval);
+ int index = Collections.binarySearch(reportLines, row, aggComparator);
+ if (index < 0) {
+ index = -index;
+ if (index > reportLines.size()) {
+ reportLines.add(row);
+ } else {
+ reportLines.add(index - 1, row);
+ }
+ } else {
+ ReportLine found = reportLines.get(index);
+ found.setUsage(found.getUsage() + row.getUsage());
+ found.setCost(found.getCost() + row.getCost());
+ return found;
+ }
+ }
+
+ return row;
+ }
+
+ /**
+ * Truncate given date for aggregates.
+ *
+ * @param date the date.
+ * @return truncated date.
+ */
+ private String truncDate(String date) {
+ if (date == null || date.length() <= truncateDateLength) {
+ return date;
+ }
+ return date.substring(0, truncateDateLength);
+ }
+
+ /**
+ * Returns the number of the report lines in list.
+ */
+ public int size() {
+ return reportLines.size();
+ }
+
+ /**
+ * Returns the report line.
+ *
+ * @param index index of the report line.
+ */
+ public ReportLine get(int index) {
+ return reportLines.get(index);
+ }
+
+ /**
+ * Removes all of the elements from list.
+ */
+ public void clear() {
+ reportLines.clear();
+ }
+
+ /**
+ * Comparator for aggregation.
+ */
+ private class AggComparator implements Comparator<ReportLine> {
+ @Override
+ public int compare(ReportLine o1, ReportLine o2) {
+ if (o1 == null) {
+ return (o2 == null ? 0 : -1);
+ } else if (o2 == null) {
+ return 1;
+ }
+
+ int result = StringUtils.compare(o1.getResourceId(), o2.getResourceId());
+ if (result == 0) {
+ result = StringUtils.compare(o1.getUsageType(), o2.getUsageType());
+ if (result == 0) {
+ result = StringUtils.compare(o1.getUsageDate(), o2.getUsageDate());
+ if (result == 0) {
+ result = StringUtils.compare(o1.getProduct(), o2.getProduct());
+ if (result == 0) {
+ result = StringUtils.compare(o1.getUser(), o2.getUser());
+ if (result == 0) {
+ return StringUtils.compare(o1.getDatalabId(), o2.getDatalabId());
+ }
+ }
+ }
+ }
+ }
+ return result;
+ }
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/aggregate/UsageDataList.java b/services/billing-aws/src/main/java/com/epam/datalab/core/aggregate/UsageDataList.java
new file mode 100644
index 0000000..f45381c
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/aggregate/UsageDataList.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core.aggregate;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * List of usage dates the billing report data.
+ */
+public class UsageDataList implements Iterable<String> {
+ /**
+ * List of dates.
+ */
+ private final Map<String, Boolean> map = new HashMap<>();
+
+ /**
+ * Appends the date to the list and returns it.
+ *
+ * @param usageDate the date of data.
+ * @return Instance of the range.
+ */
+ public void append(String usageDate) {
+ synchronized (this) {
+ if (!map.containsKey(usageDate)) {
+ map.put(usageDate, false);
+ }
+ }
+ }
+
+ /**
+ * Returns the number of the range in list.
+ */
+ public int size() {
+ return map.size();
+ }
+
+ /**
+ * Returns the value for date.
+ *
+ * @param usageDate the date.
+ */
+ public Boolean get(String usageDate) {
+ return map.get(usageDate);
+ }
+
+ /**
+ * Set the value of usageDate.
+ *
+ * @param usageDate the date.
+ */
+ public void set(String usageDate, boolean value) {
+ if (map.containsKey(usageDate)) {
+ map.put(usageDate, value);
+ }
+ }
+
+ /**
+ * Removes all of the elements from list.
+ */
+ public void clear() {
+ map.clear();
+ }
+
+ @Override
+ public Iterator<String> iterator() {
+ return map.keySet().iterator();
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ColumnInfo.java b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ColumnInfo.java
new file mode 100644
index 0000000..ee5aceb
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ColumnInfo.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core.parser;
+
+/**
+ * Describe the info of column mapping: source to target.
+ */
+public class ColumnInfo {
+
+ /**
+ * The target column name.
+ */
+ public final String targetName;
+
+ /**
+ * The source column name.
+ */
+ public final String sourceName;
+
+ /**
+ * The source column index.
+ */
+ public final int sourceIndex;
+
+ /**
+ * Instantiate the info of column mapping.
+ *
+ * @param targetName the target column name.
+ * @param sourceName the source column name.
+ * @param sourceIndex the source column index.
+ */
+ public ColumnInfo(String targetName, String sourceName, int sourceIndex) {
+ this.targetName = targetName;
+ this.sourceName = sourceName;
+ this.sourceIndex = sourceIndex;
+ }
+
+ @Override
+ public String toString() {
+ return targetName + "=" + (sourceName == null ? "" : sourceName + "[" + sourceIndex + "]");
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ColumnMeta.java b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ColumnMeta.java
new file mode 100644
index 0000000..63172bd
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ColumnMeta.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core.parser;
+
+import com.epam.datalab.exceptions.InitializationException;
+import com.epam.datalab.model.aws.ReportLine;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.MoreObjects.ToStringHelper;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Provides column meta information.
+ */
+public class ColumnMeta {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ColumnMeta.class);
+
+ /**
+ * Character for separate the tag values and column names.
+ */
+ public static final char TAG_SEPARATOR = ',';
+ /**
+ * Character for separate the column mapping.
+ */
+ public static final char MAPPING_COLUMN_SEPARATOR = ';';
+
+ /**
+ * The column names for common format.
+ */
+ static final String[] COLUMN_NAMES = {
+ ReportLine.FIELD_DATALAB_ID,
+ ReportLine.FIELD_USER_ID,
+ ReportLine.FIELD_USAGE_DATE,
+ ReportLine.FIELD_PRODUCT,
+ ReportLine.FIELD_USAGE_TYPE,
+ ReportLine.FIELD_USAGE,
+ ReportLine.FIELD_COST,
+ ReportLine.FIELD_CURRENCY_CODE,
+ ReportLine.FIELD_RESOURCE_ID,
+ ReportLine.FIELD_TAGS
+ };
+
+ /**
+ * The index of the first column for tags.
+ */
+ public static final int TAG_COLUMN_INDEX = COLUMN_NAMES.length - 1;
+
+ /**
+ * The list of target column names.
+ */
+ private List<String> targetColumnNames;
+
+ /**
+ * The list of source column names.
+ */
+ private final List<String> sourceColumnNames;
+
+ /**
+ * The list of column mapping: source to target.
+ */
+ private List<ColumnInfo> columnMapping;
+
+
+ /**
+ * Instantiate the common format parser. <b>columnMappingString</b> is semicolon separated
+ * string with key=value as target=source columns name or indexes of source column. For example,<br>
+ * "accountId=PayerAccountId;usageIntervalStart=UsageStartDate;usageIntervalEnd=UsageEndDate; ...
+ * ;tags=user:tag1,user:tag2,user:tagN".
+ *
+ * @param columnMappingString column mapping: source to target. if <b>null</b>
+ * the source data will be converted without mapping.
+ * @param sourceColumnNames the source column names.
+ * @throws InitializationException
+ */
+ public ColumnMeta(String columnMappingString, List<String> sourceColumnNames) throws InitializationException {
+ this.sourceColumnNames = sourceColumnNames;
+ if (columnMappingString != null) {
+ try {
+ setColumnMapping(columnMappingString, sourceColumnNames);
+ } catch (Exception e) {
+ throw new InitializationException("Column mapping error. " + e.getLocalizedMessage(), e);
+ }
+ }
+ }
+
+
+ /**
+ * Return the list of target column names.
+ */
+ public List<String> getTargetColumnNames() {
+ return targetColumnNames;
+ }
+
+ /**
+ * Return the list of source column names.
+ */
+ public List<String> getSourceColumnNames() {
+ return sourceColumnNames;
+ }
+
+ /**
+ * Return the list of column mapping: source to target.
+ */
+ public List<ColumnInfo> getColumnMapping() {
+ return columnMapping;
+ }
+
+
+ /**
+ * Return the index of column in the list <b>columnNames</b> or throw exception {@link InitializationException}
+ *
+ * @param columnName the name of column.
+ * @param columnNames the list of column names.
+ * @return the index of column.
+ * @throws InitializationException if column not found in the list of columns.
+ */
+ public static int getColumnIndexByName(String columnName, List<String> columnNames) throws
+ InitializationException {
+ for (int i = 0; i < columnNames.size(); i++) {
+ if (columnName.equals(columnNames.get(i))) {
+ return i;
+ }
+ }
+ throw new InitializationException("Column index not detected for column \"" + columnName + "\"");
+ }
+
+ /**
+ * Return the index of column in the list <b>columnNames</b> or throw exception {@link InitializationException}.
+ * columnName may be present as column index. For example like this "$2" for second column.
+ *
+ * @param columnName the name of column or index.
+ * @param columnNames the list of column names.
+ * @return the index of column.
+ * @throws InitializationException if column not found in the list of columns.
+ */
+ private static int getColumnIndex(String columnName, List<String> columnNames) throws InitializationException {
+ if (columnName.startsWith("$")) {
+ try {
+ return Integer.parseInt(columnName.substring(1)) - 1;
+ } catch (NumberFormatException e) {
+ // Not a column index but column name
+ }
+ }
+ if (columnNames == null) {
+ throw new InitializationException("Invalid column index \"" + columnName + "\"");
+ }
+ return getColumnIndexByName(columnName, columnNames);
+ }
+
+ /**
+ * Return the index of column in the list <b>columnNames</b> or throw exception {@link InitializationException}.
+ * columnName may be present as column index. For example like this "$2" for second column.
+ *
+ * @param columnName the name of column or index.
+ * @return the index of column.
+ * @throws InitializationException if column not found in the list of columns.
+ */
+ private static int getColumnIndex(String columnName) throws InitializationException {
+ ArrayList<String> list = new ArrayList<>(COLUMN_NAMES.length);
+ for (String s : COLUMN_NAMES) {
+ list.add(s);
+ }
+ return getColumnIndexByName(columnName, list);
+ }
+
+ /**
+ * Create map of target and source columns for column mapping. Key of map is target column, the value
+ * is source column. <b>columnMappingString</b> is semicolon separated string with key=value as
+ * target=source columns name or indexes of source column. For example,<br>
+ * "accountId=PayerAccountId;usageIntervalStart=UsageStartDate;usageIntervalEnd=UsageEndDate; ...
+ * ;tags=user:tag1,user:tag2,user:tagN".
+ *
+ * @param columnMappingString column mapping: source to target.
+ * @param sourceColumnNames
+ * @return Map of target and source columns for column mapping.
+ * @throws InitializationException
+ */
+ private Map<String, String> getSourceToTarget(String columnMappingString, List<String> sourceColumnNames) throws
+ InitializationException {
+ String[] entries = StringUtils.split(columnMappingString, MAPPING_COLUMN_SEPARATOR);
+ Map<String, String> sourceToTarget = new HashMap<>();
+
+ for (String entry : entries) {
+ if (entry.trim().isEmpty() || !entry.contains("=")) {
+ throw new InitializationException("Invalid the entry \"" + entry + "\"in column mapping");
+ }
+ String[] pair = StringUtils.split(entry, '=');
+ if (pair.length != 2) {
+ throw new InitializationException("Invalid the entry \"" + entry + "\"in column mapping");
+ }
+
+ pair[0] = pair[0].trim();
+ pair[1] = pair[1].trim();
+
+ try {
+ int index = getColumnIndex(pair[0]);
+ pair[0] = COLUMN_NAMES[index];
+ } catch (InitializationException e) {
+ throw new InitializationException("Unkown target column \"" + pair[0] + "\".", e);
+ }
+
+ try {
+ if (!pair[0].equals(ReportLine.FIELD_TAGS)) {
+ int index = getColumnIndex(pair[1], sourceColumnNames);
+ if (sourceColumnNames != null) {
+ pair[1] = sourceColumnNames.get(index);
+ }
+ }
+ } catch (InitializationException e) {
+ if (sourceColumnNames == null) {
+ throw new InitializationException("Invalid column index \"" + pair[1] + "\" or column header not " +
+ "defined");
+ }
+ throw new InitializationException("Unkown source column \"" + pair[1] + "\".", e);
+ }
+ sourceToTarget.put(pair[0], pair[1]);
+ }
+
+ return sourceToTarget;
+ }
+
+ /**
+ * Initialize and set column mapping. <b>columnMappingString</b> is semicolon separated string with key=value as
+ * target=source columns name or indexes of source column. For example,<br>
+ * "accountId=PayerAccountId;usageIntervalStart=UsageStartDate;usageIntervalEnd=UsageEndDate; ...
+ * ;tags=user:tag1,user:tag2,user:tagN".
+ *
+ * @param columnMappingString column mapping: source to target. if <b>null</b>
+ * the source data will be converted without mapping.
+ * @param sourceColumnNames the list of source column names.
+ * @throws InitializationException
+ */
+ private void setColumnMapping(String columnMappingString, List<String> sourceColumnNames) throws
+ InitializationException {
+ if (columnMappingString == null) {
+ throw new InitializationException("Mapping not defined.");
+ }
+
+ Map<String, String> sourceToTarget = getSourceToTarget(columnMappingString, sourceColumnNames);
+ String tags = sourceToTarget.get(ReportLine.FIELD_TAGS);
+ List<String> tagColumns = (tags == null ? null :
+ Arrays.asList(StringUtils.split(tags, TAG_SEPARATOR)));
+
+ LOGGER.info("Mapping columns [target=source:name[index]]:");
+ int columnCount = COLUMN_NAMES.length - 1;
+ int tagCount = (tagColumns == null ? 0 : tagColumns.size());
+ columnMapping = new ArrayList<>(columnCount + tagCount);
+ targetColumnNames = new ArrayList<>(columnCount + tagCount);
+
+ for (int i = 0; i < columnCount; i++) {
+ String sourceName = sourceToTarget.get(COLUMN_NAMES[i]);
+ ColumnInfo columnInfo = new ColumnInfo(
+ COLUMN_NAMES[i],
+ sourceName,
+ (sourceName == null ? -1 : getColumnIndex(sourceName, sourceColumnNames)));
+ columnMapping.add(columnInfo);
+ targetColumnNames.add(COLUMN_NAMES[i]);
+ LOGGER.info(" " + columnInfo.toString());
+ }
+
+ for (int i = 0; i < tagCount; i++) {
+ String sourceName = tagColumns.get(i).trim();
+ int sourceIndex = getColumnIndex(sourceName, sourceColumnNames);
+ if (sourceColumnNames != null) {
+ sourceName = sourceColumnNames.get(sourceIndex);
+ }
+ ColumnInfo columnInfo = new ColumnInfo(
+ ReportLine.FIELD_TAGS,
+ sourceName,
+ sourceIndex);
+ columnMapping.add(columnInfo);
+ targetColumnNames.add(sourceName);
+ LOGGER.info(" " + columnInfo.toString());
+ }
+ }
+
+
+ /**
+ * Returns a string representation of the object.
+ *
+ * @param self the object to generate the string for (typically this), used only for its class name.
+ */
+ public ToStringHelper toStringHelper(Object self) {
+ return MoreObjects.toStringHelper(self)
+ .add("targetColumnNames", targetColumnNames)
+ .add("sourceColumnNames", sourceColumnNames)
+ .add("columnMapping", columnMapping);
+ }
+
+ @Override
+ public String toString() {
+ return toStringHelper(this).toString();
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/parser/CommonFormat.java b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/CommonFormat.java
new file mode 100644
index 0000000..28b4242
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/CommonFormat.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core.parser;
+
+import com.epam.datalab.exceptions.ParseException;
+import com.epam.datalab.model.aws.ReportLine;
+import org.apache.commons.lang3.StringUtils;
+
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+/**
+ * Provides common format features.
+ */
+public class CommonFormat {
+ /**
+ * Character for separate field names and values.
+ */
+ public static final char FIELD_SEPARATOR = ',';
+
+ /**
+ * Character for termination field names and values.
+ */
+ public static final char FIELD_DELIMITER = '"';
+
+ /**
+ * Escape character.
+ */
+ public static final char ESCAPE_CHAR = '\\';
+
+ /**
+ * Default character used for decimal sign.
+ */
+ public static final char DECIMAL_SEPARATOR_DEFAULT = '.';
+
+ /**
+ * Default character used for thousands separator.
+ */
+ public static final char DECIMAL_GROUPING_SEPARATOR_DEFAULT = ' ';
+
+ /**
+ * String of the field separator for replacement to target data.
+ */
+ private static final String DELIMITER_REPLACE_FROM = String.valueOf(FIELD_DELIMITER);
+
+ /**
+ * String of the escaped field separator for replacement to target data.
+ */
+ private static final String DELIMITER_REPLACE_TO = new StringBuilder()
+ .append(ESCAPE_CHAR)
+ .append(FIELD_DELIMITER)
+ .toString();
+
+ /**
+ * Formatter for convert decimal numbers to string.
+ */
+ private static final DecimalFormat DECIMAL_TO_STRING_FORMAT;
+
+ static {
+ DECIMAL_TO_STRING_FORMAT = new DecimalFormat();
+ DecimalFormatSymbols symbols = new DecimalFormatSymbols();
+ symbols.setDecimalSeparator(DECIMAL_SEPARATOR_DEFAULT);
+ DECIMAL_TO_STRING_FORMAT.setDecimalFormatSymbols(symbols);
+ DECIMAL_TO_STRING_FORMAT.setGroupingUsed(false);
+ DECIMAL_TO_STRING_FORMAT.setMaximumFractionDigits(100);
+ }
+
+
+ /**
+ * Column meta information.
+ */
+ private final ColumnMeta columnMeta;
+
+ /**
+ * Formatter for parse of decimal number .
+ */
+ private final DecimalFormat sourceDecimalFormat;
+
+
+ /**
+ * Instantiate the helper for common format.
+ *
+ * @param columnMeta column meta information.
+ */
+ public CommonFormat(ColumnMeta columnMeta) {
+ this(columnMeta, DECIMAL_SEPARATOR_DEFAULT, DECIMAL_GROUPING_SEPARATOR_DEFAULT);
+ }
+
+ /**
+ * Instantiate the helper for common format.
+ *
+ * @param columnMeta column meta information.
+ * @param decimalSeparator the character used for decimal sign.
+ * @param groupingSeparator the character used for thousands separator.
+ */
+ public CommonFormat(ColumnMeta columnMeta, char sourceDecimalSeparator, char sourceGroupingSeparator) {
+ this.columnMeta = columnMeta;
+ this.sourceDecimalFormat = getDecimalFormat(sourceDecimalSeparator, sourceGroupingSeparator);
+ }
+
+
+ /**
+ * Create and return the target row for common format from source.
+ *
+ * @param sourceRow the source row.
+ * @return row in common format.
+ * @throws ParseException
+ */
+ public ReportLine toCommonFormat(List<String> sourceRow) throws ParseException {
+ if (columnMeta.getColumnMapping() == null) {
+ return toReportLine(sourceRow);
+ }
+
+ List<String> targetRow = new ArrayList<>();
+ for (ColumnInfo columnInfo : columnMeta.getColumnMapping()) {
+ targetRow.add((columnInfo.sourceIndex < 0 ? "" :
+ (columnInfo.sourceIndex < sourceRow.size() ? sourceRow.get(columnInfo.sourceIndex) : null)));
+ }
+ return toReportLine(targetRow);
+ }
+
+ /**
+ * Add the column value to string in CSV format.
+ *
+ * @param sb the buffer of sting.
+ * @param value the value.
+ * @return the buffer of string.
+ */
+ private static StringBuilder addToStringBuilder(StringBuilder sb, String value) {
+ if (sb.length() > 0) {
+ sb.append(FIELD_SEPARATOR);
+ }
+ return sb.append(FIELD_DELIMITER)
+ .append(StringUtils.replace(value, DELIMITER_REPLACE_FROM, DELIMITER_REPLACE_TO))
+ .append(FIELD_DELIMITER);
+ }
+
+ /**
+ * Convert row to line in CSV format.
+ *
+ * @param row row for convertation.
+ * @return string in CSV format.
+ */
+ public static String rowToString(List<String> row) {
+ StringBuilder sb = new StringBuilder();
+ for (String s : row) {
+ addToStringBuilder(sb, s);
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Convert row to line in CSV format.
+ *
+ * @param row row for convertation.
+ * @return string in CSV format.
+ */
+ public static String rowToString(ReportLine row) {
+ StringBuilder sb = new StringBuilder();
+
+ addToStringBuilder(sb, row.getDatalabId());
+ addToStringBuilder(sb, row.getUser());
+ addToStringBuilder(sb, row.getUsageDate());
+ addToStringBuilder(sb, row.getProduct());
+ addToStringBuilder(sb, row.getUsageType());
+ addToStringBuilder(sb, doubleToString(row.getUsage()));
+ addToStringBuilder(sb, doubleToString(row.getCost()));
+ addToStringBuilder(sb, row.getCurrencyCode());
+ addToStringBuilder(sb, row.getResourceType().toString());
+ addToStringBuilder(sb, row.getResourceId());
+
+ if (row.getTags() != null) {
+ for (String key : row.getTags().keySet()) {
+ addToStringBuilder(sb, row.getTags().get(key));
+ }
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Create and return decimal formatter.
+ *
+ * @param decimalSeparator the character used for decimal sign.
+ * @param groupingSeparator the character used for thousands separator.
+ * @return Formatter for decimal digits.
+ */
+ private DecimalFormat getDecimalFormat(char decimalSeparator, char groupingSeparator) {
+ DecimalFormat df = new DecimalFormat();
+ DecimalFormatSymbols symbols = new DecimalFormatSymbols();
+ symbols.setDecimalSeparator(decimalSeparator);
+ symbols.setGroupingSeparator(groupingSeparator);
+ df.setDecimalFormatSymbols(symbols);
+ return df;
+ }
+
+ /**
+ * Parse and return double value. If value is <b>null</b> or empty return zero.
+ *
+ * @param columnName the name of column.
+ * @param value the value.
+ * @throws ParseException
+ */
+ public double parseDouble(String columnName, String value) throws ParseException {
+ if (value == null || value.trim().isEmpty()) {
+ return 0;
+ }
+ try {
+ return sourceDecimalFormat.parse(value).doubleValue();
+ } catch (Exception e) {
+ throw new ParseException("Cannot cast column " + columnName + " value \"" + value + "\" to double: " + e
+ .getLocalizedMessage(), e);
+ }
+ }
+
+ /**
+ * Return the string representation of double value.
+ *
+ * @param value the value.
+ */
+ public static String doubleToString(double value) {
+ return DECIMAL_TO_STRING_FORMAT.format(value);
+ }
+
+ /**
+ * Creates and returns the line of billing report from the list of values.
+ *
+ * @param row the list of values.
+ * @return the line of billing report.
+ * @throws ParseException
+ */
+ public ReportLine toReportLine(List<String> row) throws ParseException {
+ if (row.size() < ColumnMeta.TAG_COLUMN_INDEX || row.size() > columnMeta.getTargetColumnNames().size()) {
+ throw new ParseException("Invalid the number of columns in list: expected from " + ColumnMeta
+ .TAG_COLUMN_INDEX +
+ " to " + columnMeta.getTargetColumnNames().size() + ", actual " + row.size());
+ }
+ ReportLine line = new ReportLine();
+ int i = 0;
+
+ line.setDatalabId(row.get(i));
+ line.setUser(row.get(++i));
+ line.setUsageDate(row.get(++i));
+ line.setProduct(row.get(++i));
+ line.setUsageType(row.get(++i));
+ line.setUsage(parseDouble("usage", row.get(++i)));
+ line.setCost(parseDouble("cost", row.get(++i)));
+ line.setCurrencyCode(row.get(++i));
+ line.setResourceTypeId(row.get(++i));
+
+ if (row.size() >= ColumnMeta.TAG_COLUMN_INDEX) {
+ LinkedHashMap<String, String> tags = new LinkedHashMap<>();
+ i++;
+ while (i < row.size()) {
+ tags.put(columnMeta.getTargetColumnNames().get(i), row.get(i++));
+ }
+ line.setTags(tags);
+ }
+
+ return line;
+ }
+
+
+ /**
+ * Print row to console.
+ *
+ * @param row array of values.
+ */
+ public static void printRow(String[] row) {
+ System.out.print(" | ");
+ for (String s : row) {
+ System.out.print(s + " | ");
+ }
+ System.out.println();
+ }
+
+ /**
+ * Print row to console.
+ *
+ * @param row list of values.
+ */
+ public static void printRow(List<String> row) {
+ printRow(row.toArray(new String[row.size()]));
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ConditionEvaluate.java b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ConditionEvaluate.java
new file mode 100644
index 0000000..abfd31e
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ConditionEvaluate.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core.parser;
+
+import com.epam.datalab.exceptions.InitializationException;
+import com.epam.datalab.exceptions.ParseException;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.MoreObjects.ToStringHelper;
+import org.apache.commons.jexl3.JexlBuilder;
+import org.apache.commons.jexl3.JexlContext;
+import org.apache.commons.jexl3.JexlEngine;
+import org.apache.commons.jexl3.MapContext;
+import org.apache.commons.jexl3.internal.Script;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Evaluate condition for filtering source data.
+ */
+public class ConditionEvaluate {
+
+ /**
+ * Names of columns for condition.
+ */
+ private String[] columnNames;
+
+ /**
+ * Indexes of columns for condition.
+ */
+ private int[] columnIndexes;
+
+ /**
+ * JEXL expression for evaluation.
+ */
+ private final Script expression;
+
+ /**
+ * JEXL context to evaluate row.
+ */
+ private final JexlContext jexlContext;
+
+ /**
+ * Instantiate the engine to evaluate condition.
+ *
+ * @param columnNames the list of column names.
+ * @param condition condition for filtering data.
+ * @throws InitializationException
+ */
+ public ConditionEvaluate(List<String> columnNames, String condition) throws InitializationException {
+ //Replace : to . in column names
+ List<String> colNames = new ArrayList<>(columnNames.size());
+ for (int i = 0; i < columnNames.size(); i++) {
+ String name = columnNames.get(i);
+ if (name.indexOf(':') > -1 && condition.indexOf(name) > -1) {
+ String newName = StringUtils.replaceChars(name, ':', '.');
+ colNames.add(newName);
+ condition = StringUtils.replace(condition, name, newName);
+ } else {
+ colNames.add(name);
+ }
+ }
+
+ try {
+ JexlEngine engine = new JexlBuilder().strict(true).silent(false).debug(true).create();
+ expression = (Script) engine.createExpression(condition);
+ jexlContext = new MapContext();
+ } catch (Exception e) {
+ throw new InitializationException("Cannot initialize JEXL engine for condition: " + condition + ". " +
+ e.getLocalizedMessage(), e);
+ }
+
+ // Create mapping of columns for evaluations.
+ List<String> names = new ArrayList<>();
+ List<Integer> indexes = new ArrayList<>();
+ for (List<String> variableList : expression.getVariables()) {
+ String columnName = StringUtils.join(variableList, '.');
+ int index = getColumnIndex(colNames, columnName);
+ if (index == -1) {
+ throw new InitializationException("Unknow source column name \"" + columnName + "\" in condition: " +
+ expression.getSourceText() + ". Known column names: " + StringUtils.join(columnNames, ", ") + ".");
+ }
+ names.add(columnName);
+ indexes.add(index);
+ }
+
+ this.columnNames = new String[names.size()];
+ this.columnIndexes = new int[indexes.size()];
+ for (int i = 0; i < indexes.size(); i++) {
+ this.columnNames[i] = names.get(i);
+ this.columnIndexes[i] = indexes.get(i);
+ }
+ }
+
+ /**
+ * Find and return the index of column in the given column list.
+ *
+ * @param columnNames the list of column names.
+ * @param columnName the name of column to find.
+ */
+ private int getColumnIndex(List<String> columnNames, String columnName) {
+ for (int i = 0; i < columnNames.size(); i++) {
+ if (columnName.equals(columnNames.get(i))) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Evaluate condition for given row.
+ *
+ * @param row the row to evaluate.
+ * @return <true> if condition is true.
+ * @throws ParseException if condition is not return boolean type.
+ */
+ public boolean evaluate(List<String> row) throws ParseException {
+ for (int i = 0; i < columnNames.length; i++) {
+ jexlContext.set(columnNames[i], row.get(columnIndexes[i]));
+ }
+ Object value;
+ try {
+ value = expression.evaluate(jexlContext);
+ } catch (Exception e) {
+ throw new ParseException("Cannot evaluate condition: " + expression.getSourceText() + ". " +
+ e.getLocalizedMessage(), e);
+ }
+ if (value instanceof Boolean) {
+ return (Boolean) value;
+ }
+ throw new ParseException("Invalid condition: " + expression.getSourceText());
+ }
+
+
+ /**
+ * Returns a string representation of the object.
+ *
+ * @param self the object to generate the string for (typically this), used only for its class name.
+ */
+ public ToStringHelper toStringHelper(Object self) {
+ return MoreObjects.toStringHelper(self)
+ .add("columnNames", columnNames)
+ .add("columnIndexes", columnIndexes);
+ }
+
+ @Override
+ public String toString() {
+ return toStringHelper(this).toString();
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ParserBase.java b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ParserBase.java
new file mode 100644
index 0000000..3b54850
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ParserBase.java
@@ -0,0 +1,395 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core.parser;
+
+import com.epam.datalab.core.AdapterBase;
+import com.epam.datalab.core.FilterBase;
+import com.epam.datalab.core.ModuleBase;
+import com.epam.datalab.core.aggregate.AggregateGranularity;
+import com.epam.datalab.core.aggregate.DataAggregator;
+import com.epam.datalab.exceptions.AdapterException;
+import com.epam.datalab.exceptions.InitializationException;
+import com.epam.datalab.exceptions.ParseException;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.MoreObjects.ToStringHelper;
+import org.apache.commons.lang3.StringUtils;
+import org.bson.Document;
+
+import javax.validation.constraints.NotNull;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Abstract module of parser.<br>
+ * See description of {@link ModuleBase} how to create your own parser.
+ */
+public abstract class ParserBase extends ModuleBase {
+
+ /**
+ * Default character used for decimal sign.
+ */
+ public static final char DECIMAL_SEPARATOR_DEFAULT = '.';
+
+ /**
+ * Default character used for thousands separator.
+ */
+ public static final char DECIMAL_GROUPING_SEPARATOR_DEFAULT = ' ';
+
+ /**
+ * Name of key for date report data.
+ */
+ public static final String DATA_KEY_START_DATE = "ParserBase.maxStartDate";
+
+
+ /**
+ * Mapping columns from source format to target.
+ */
+ @JsonProperty
+ private String columnMapping = null;
+
+ /**
+ * Where condition for filtering the source data.
+ */
+ @JsonProperty
+ private String whereCondition = null;
+
+ /**
+ * How to aggregate the parsed data.
+ */
+ @JsonProperty
+ @NotNull
+ private AggregateGranularity aggregate = AggregateGranularity.NONE;
+
+ /**
+ * Character used for decimal sign of source data.
+ */
+ @JsonProperty
+ @NotNull
+ private char decimalSeparator = DECIMAL_SEPARATOR_DEFAULT;
+
+ /**
+ * Character used for thousands separator of source data.
+ */
+ @JsonProperty
+ @NotNull
+ private char groupingSeparator = DECIMAL_GROUPING_SEPARATOR_DEFAULT;
+
+
+ /**
+ * Adapter for reading source data.
+ */
+ @JsonIgnore
+ private AdapterBase adapterIn;
+
+ /**
+ * Adapter for writing converted data.
+ */
+ @JsonIgnore
+ private AdapterBase adapterOut;
+
+ /**
+ * Filter for source and converted data.
+ */
+ @JsonIgnore
+ private FilterBase filter;
+
+
+ /**
+ * Column meta information.
+ */
+ @JsonIgnore
+ private ColumnMeta columnMeta;
+
+ /**
+ * Condition for filtering the source data.
+ */
+ @JsonIgnore
+ private ConditionEvaluate condition;
+
+ /**
+ * Aggregator of billing report.
+ */
+ @JsonIgnore
+ private DataAggregator aggregator;
+
+ /**
+ * Common format helper.
+ */
+ @JsonIgnore
+ private CommonFormat commonFormat;
+
+ /**
+ * Parser statistics.
+ */
+ @JsonIgnore
+ private final List<ParserStatistics> statistics = new ArrayList<>();
+
+ /**
+ * Current parser statistics.
+ */
+ @JsonIgnore
+ ParserStatistics currentStatistics = null;
+
+
+ /**
+ * Return mapping columns from source format to target.
+ */
+ public String getColumnMapping() {
+ return columnMapping;
+ }
+
+ /**
+ * Set mapping columns from source format to target.
+ */
+ public void setColumnMapping(String columnMapping) {
+ this.columnMapping = columnMapping;
+ }
+
+ /**
+ * Return where condition for filtering the source data.
+ */
+ public String getWhereCondition() {
+ return whereCondition;
+ }
+
+ /**
+ * Set where condition for filtering the source data.
+ */
+ public void setWhereCondition(String whereCondition) {
+ this.whereCondition = whereCondition;
+ }
+
+ /**
+ * Return how to aggregate the parsed data.
+ */
+ public AggregateGranularity getAggregate() {
+ return aggregate;
+ }
+
+ /**
+ * Set how to aggregate the parsed data.
+ *
+ * @throws InitializationException
+ */
+ public void setAggregate(String aggregate) throws InitializationException {
+ if (aggregate == null) {
+ throw new InitializationException("Property aggregate cannot be null");
+ }
+ AggregateGranularity value = AggregateGranularity.of(aggregate);
+ if (value == null) {
+ throw new InitializationException("Invalid value \"" + aggregate + "\" for property aggregate. " +
+ "Should be one of: " + StringUtils.join(AggregateGranularity.values(), ", "));
+ }
+ this.aggregate = value;
+ }
+
+ /**
+ * Return character used for decimal sign of source data.
+ */
+ public char getDecimalSeparator() {
+ return decimalSeparator;
+ }
+
+ /**
+ * Set character used for decimal sign of source data.
+ */
+ public void setDecimalSeparator(char decimalSeparator) {
+ this.decimalSeparator = decimalSeparator;
+ }
+
+ /**
+ * Return character used for thousands separator of source data.
+ */
+ public char getGroupingSeparator() {
+ return groupingSeparator;
+ }
+
+ /**
+ * Set character used for thousands separator of source data.
+ */
+ public void setGroupingSeparator(char groupingSeparator) {
+ this.groupingSeparator = groupingSeparator;
+ }
+
+
+ /**
+ * Return the adapter for reading source data.
+ */
+ public AdapterBase getAdapterIn() {
+ return adapterIn;
+ }
+
+ /**
+ * Return the adapter for writing converted data.
+ */
+ public AdapterBase getAdapterOut() {
+ return adapterOut;
+ }
+
+ /**
+ * Return the filter for source and converted data.
+ */
+ public FilterBase getFilter() {
+ return filter;
+ }
+
+ /**
+ * Return the column meta information.
+ */
+ public ColumnMeta getColumnMeta() {
+ return columnMeta;
+ }
+
+ /**
+ * Return the condition for filtering the source data.
+ */
+ public ConditionEvaluate getCondition() {
+ return condition;
+ }
+
+ /**
+ * Return the aggregator of billing report.
+ */
+ public DataAggregator getAggregator() {
+ return aggregator;
+ }
+
+ /**
+ * Return the common format helper.
+ */
+ public CommonFormat getCommonFormat() {
+ return commonFormat;
+ }
+
+ /**
+ * Return the parser statistics.
+ */
+ public List<ParserStatistics> getStatistics() {
+ return statistics;
+ }
+
+ /**
+ * Add and return the new instance for statistics.
+ *
+ * @param entryName the name of new entry.
+ */
+ public ParserStatistics addStatistics(String entryName) {
+ currentStatistics = new ParserStatistics(entryName);
+ statistics.add(currentStatistics);
+ return currentStatistics;
+ }
+
+ /**
+ * Return the current parser statistics.
+ */
+ public ParserStatistics getCurrentStatistics() {
+ return currentStatistics;
+ }
+
+
+ /**
+ * Initialize the parser.
+ *
+ * @throws InitializationException
+ */
+ public abstract void initialize() throws InitializationException;
+
+ /**
+ * Parse the source data to common format and write it to output adapter.
+ *
+ * @return
+ * @throws InitializationException
+ * @throws AdapterException
+ * @throws ParseException
+ */
+ public abstract List<Document> parse() throws InitializationException, AdapterException, ParseException;
+
+ /**
+ * Build parser from given modules.
+ *
+ * @param adapterIn the adapter for reading source data.
+ * @param adapterOut the adapter for writing converted data.
+ * @param filter the filter for source and converted data. May be <b>null<b>.
+ */
+ public ParserBase build(AdapterBase adapterIn, AdapterBase adapterOut, FilterBase filter) {
+ this.adapterIn = adapterIn;
+ this.adapterOut = adapterOut;
+ if (filter != null) {
+ filter.setParser(this);
+ }
+ this.filter = filter;
+ return this;
+ }
+
+
+ /**
+ * Initialize ParserBase.
+ *
+ * @param header - the header of source data.
+ * @throws InitializationException
+ * @throws AdapterException
+ */
+ protected void init(List<String> header) throws InitializationException, AdapterException {
+ columnMeta = new ColumnMeta(columnMapping, header);
+ if (whereCondition != null) {
+ if (columnMeta.getSourceColumnNames() == null) {
+ throw new InitializationException("To use the whereCondition property you must specify and have the header of source data");
+ }
+ condition = new ConditionEvaluate(columnMeta.getSourceColumnNames(), whereCondition);
+ } else {
+ condition = null;
+ }
+ commonFormat = new CommonFormat(columnMeta, decimalSeparator, groupingSeparator);
+
+ if (aggregate != AggregateGranularity.NONE) {
+ aggregator = new DataAggregator(aggregate);
+ }
+
+ if (getAdapterOut().isWriteHeader()) {
+ getAdapterOut().writeHeader(columnMeta.getTargetColumnNames());
+ }
+ }
+
+
+ /**
+ * Return the index of source column by column name.
+ *
+ * @param columnName the name of column.
+ * @throws InitializationException
+ */
+ public int getSourceColumnIndexByName(String columnName) throws InitializationException {
+ return ColumnMeta.getColumnIndexByName(columnName, columnMeta.getSourceColumnNames());
+ }
+
+
+ @Override
+ public ToStringHelper toStringHelper(Object self) {
+ return super.toStringHelper(self)
+ .add("adapterIn", (adapterIn == null ? null : adapterIn.getType()))
+ .add("adapterOut", (adapterOut == null ? null : adapterOut.getType()))
+ .add("filter", (filter == null ? null : filter.getType()))
+ .add("columnMapping", columnMapping)
+ .add("whereCondition", whereCondition)
+ .add("aggregate", aggregate)
+ .add("decimalSeparator", decimalSeparator)
+ .add("groupingSeparator", groupingSeparator);
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ParserByLine.java b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ParserByLine.java
new file mode 100644
index 0000000..11f06cf
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ParserByLine.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core.parser;
+
+import com.epam.datalab.core.ModuleBase;
+import com.epam.datalab.core.aggregate.AggregateGranularity;
+import com.epam.datalab.exceptions.AdapterException;
+import com.epam.datalab.exceptions.GenericException;
+import com.epam.datalab.exceptions.InitializationException;
+import com.epam.datalab.exceptions.ParseException;
+import com.epam.datalab.model.aws.ReportLine;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.bson.Document;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Abstract module of parser by the line.<br>
+ * See description of {@link ModuleBase} how to create your own parser.
+ */
+public abstract class ParserByLine extends ParserBase {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ParserByLine.class);
+ private static final String ENTRY_NAME = "\nEntry name: ";
+ private static final String SOURCE_LINE = "\nSource line[";
+
+ /**
+ * Parse the header of source data and return it.
+ *
+ * @return the header of source data.
+ * @throws AdapterException
+ * @throws ParseException
+ */
+ public abstract List<String> parseHeader() throws AdapterException, ParseException;
+
+ /**
+ * Parse the row from source line and return result row.
+ *
+ * @param line the source line.
+ * @return the parsed row.
+ * @throws ParseException
+ */
+ public abstract List<String> parseRow(String line) throws ParseException;
+
+ /**
+ * Read the line from adapter and return it.
+ *
+ * @return the parsed row from adapterIn.
+ * @throws AdapterException
+ */
+ @JsonIgnore
+ public String getNextRow() throws AdapterException {
+ String line = getAdapterIn().readLine();
+ if (line == null) {
+ return null;
+ }
+ getCurrentStatistics().incrRowReaded();
+ return line;
+ }
+
+ /**
+ * Initialize ParserBase.
+ *
+ * @throws InitializationException
+ * @throws AdapterException
+ * @throws ParseException
+ */
+ protected boolean init() throws InitializationException, AdapterException, ParseException {
+ getAdapterIn().open();
+ LOGGER.debug("Source data has multy entry {}", getAdapterIn().hasMultyEntry());
+ if (!initEntry()) {
+ return false;
+ }
+ getAdapterOut().open();
+ return true;
+ }
+
+ /**
+ * Initialize for each entry ParserBase.
+ *
+ * @throws InitializationException
+ * @throws AdapterException
+ * @throws ParseException
+ */
+ private boolean initEntry() throws InitializationException, AdapterException, ParseException {
+ if (getAdapterIn().hasMultyEntry() && !getAdapterIn().hasEntryData()) {
+ return false;
+ }
+ addStatistics(getAdapterIn().getEntryName());
+ getCurrentStatistics().start();
+
+ super.init(parseHeader());
+ initialize();
+ if (getFilter() != null) {
+ getFilter().initialize();
+ }
+ return true;
+ }
+
+ /**
+ * Close adapters.
+ *
+ * @throws AdapterException
+ */
+ protected void close(boolean silent) throws AdapterException {
+ AdapterException ex = null;
+ try {
+ getAdapterIn().close();
+ } catch (Exception e) {
+ if (silent) {
+ LOGGER.warn("Cannot close adapterIn. {}", e.getLocalizedMessage(), e);
+ } else {
+ ex = new AdapterException("Cannot close adapterIn. " + e.getLocalizedMessage(), e);
+ }
+ }
+ try {
+ getAdapterOut().close();
+ } catch (Exception e) {
+ if (silent || ex != null) {
+ LOGGER.warn("Cannot close adapterOut. {}", e.getLocalizedMessage(), e);
+ } else {
+ ex = new AdapterException("Cannot close adapterOut. " + e.getLocalizedMessage(), e);
+ }
+ }
+ try {
+ getModuleData().closeMongoConnection();
+ } catch (IOException e) {
+ if (silent || ex != null) {
+ LOGGER.warn("Cannot close mongo connection. {}", e.getLocalizedMessage(), e);
+ } else {
+ ex = new AdapterException("Cannot close mongo connection. " + e.getLocalizedMessage(), e);
+ }
+ }
+ if (!silent && ex != null) {
+ throw ex;
+ }
+ }
+
+ /**
+ * Parse the source data to common format and write it to output adapter.
+ *
+ * @return list of billing data
+ * @throws InitializationException
+ * @throws AdapterException
+ * @throws ParseException
+ */
+ public List<Document> parse() throws InitializationException, AdapterException, ParseException {
+ List<Document> billingData = new ArrayList<>();
+ try {
+ if (init()) {
+ String line;
+ List<String> row;
+ ReportLine reportLine;
+ LOGGER.info("Parsing {}", getAdapterIn().getEntryName());
+
+ while ((line = getNextRow()) != null) {
+ if (getFilter() != null && (line = getFilter().canParse(line)) == null) {
+ getCurrentStatistics().incrRowFiltered();
+ continue;
+ }
+
+ row = parseRow(line);
+ if ((getFilter() != null && (row = getFilter().canTransform(row)) == null)) {
+ getCurrentStatistics().incrRowFiltered();
+ continue;
+ }
+ try {
+ if (getCondition() != null && !getCondition().evaluate(row)) {
+ getCurrentStatistics().incrRowFiltered();
+ continue;
+ }
+ } catch (ParseException e) {
+ throw new ParseException(e.getLocalizedMessage() + ENTRY_NAME + getCurrentStatistics().getEntryName() +
+ SOURCE_LINE + getCurrentStatistics().getRowReaded() + "]: " + line, e);
+ } catch (Exception e) {
+ throw new ParseException("Cannot evaluate condition " + getWhereCondition() + ". " +
+ e.getLocalizedMessage() + ENTRY_NAME + getCurrentStatistics().getEntryName() +
+ SOURCE_LINE + getCurrentStatistics().getRowReaded() + "]: " + line, e);
+ }
+
+ try {
+ reportLine = getCommonFormat().toCommonFormat(row);
+ } catch (ParseException e) {
+ throw new ParseException("Cannot cast row to common format. " +
+ e.getLocalizedMessage() + ENTRY_NAME + getCurrentStatistics().getEntryName() +
+ SOURCE_LINE + getCurrentStatistics().getRowReaded() + "]: " + line, e);
+ }
+ if (getFilter() != null && (reportLine = getFilter().canAccept(reportLine)) == null) {
+ getCurrentStatistics().incrRowFiltered();
+ continue;
+ }
+
+ getCurrentStatistics().incrRowParsed();
+ if (getAggregate() != AggregateGranularity.NONE) {
+ getAggregator().append(reportLine);
+ } else {
+ billingData.add(getAdapterOut().writeRow(reportLine));
+ getCurrentStatistics().incrRowWritten();
+ }
+ }
+
+ if (getAggregate() != AggregateGranularity.NONE) {
+ for (int i = 0; i < getAggregator().size(); i++) {
+ billingData.add(getAdapterOut().writeRow(getAggregator().get(i)));
+ getCurrentStatistics().incrRowWritten();
+ }
+ }
+ }
+ } catch (GenericException e) {
+ close(true);
+ if (getCurrentStatistics() != null) {
+ getCurrentStatistics().stop();
+ }
+ throw e;
+ } catch (Exception e) {
+ close(true);
+ if (getCurrentStatistics() != null) {
+ getCurrentStatistics().stop();
+ }
+ throw new ParseException("Unknown parser error. " + e.getLocalizedMessage(), e);
+ }
+
+ close(false);
+ if (getCurrentStatistics() != null) {
+ getCurrentStatistics().stop();
+ }
+ return billingData;
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ParserStatistics.java b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ParserStatistics.java
new file mode 100644
index 0000000..966bbd0
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/core/parser/ParserStatistics.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.core.parser;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.MoreObjects.ToStringHelper;
+
+/**
+ * Store the statistic of parser processing.
+ */
+public class ParserStatistics {
+ /**
+ * Name of parsed entry.
+ */
+ private final String entryName;
+
+ /**
+ * Time is milliseconds when parser has been started.
+ */
+ private long timeStartInMillis = 0;
+
+ /**
+ * Parsing time in milliseconds.
+ */
+ private long elapsedTimeInMillis = 0;
+
+ /**
+ * Number of rows read.
+ */
+ private long rowReaded;
+
+ /**
+ * Number of rows skipped.
+ */
+ private long rowSkipped;
+
+ /**
+ * Number of rows filtered.
+ */
+ private long rowFiltered;
+
+ /**
+ * Number of rows parsed.
+ */
+ private long rowParsed;
+
+ /**
+ * Number of rows write.
+ */
+ private long rowWritten;
+
+
+ public ParserStatistics(String entryName) {
+ this.entryName = entryName;
+ }
+
+ public void start() {
+ timeStartInMillis = System.currentTimeMillis();
+ elapsedTimeInMillis = 0;
+ rowReaded = 0;
+ rowSkipped = 0;
+ rowFiltered = 0;
+ rowParsed = 0;
+ rowWritten = 0;
+ }
+
+ public void stop() {
+ if (timeStartInMillis != 0) {
+ elapsedTimeInMillis = System.currentTimeMillis() - timeStartInMillis;
+ timeStartInMillis = 0;
+ }
+ }
+
+
+ /**
+ * Return the name of parsed entry.
+ */
+ public String getEntryName() {
+ return entryName;
+ }
+
+ /**
+ * Return the elapsed time in milliseconds of initializing, reading, filtering, parsing and writing operations.
+ */
+ public long getElapsedTime() {
+ return (elapsedTimeInMillis != 0 ?
+ elapsedTimeInMillis :
+ timeStartInMillis == 0 ? 0 : System.currentTimeMillis() - timeStartInMillis);
+ }
+
+ /**
+ * Return the number of rows read.
+ */
+ public long getRowReaded() {
+ return rowReaded;
+ }
+
+ /**
+ * Return the number of rows skipped.
+ */
+ public long getRowSkipped() {
+ return rowSkipped;
+ }
+
+ /**
+ * Return the number of rows filtered.
+ */
+ public long getRowFiltered() {
+ return rowFiltered;
+ }
+
+ /**
+ * Return the number of rows parsed.
+ */
+ public long getRowParsed() {
+ return rowParsed;
+ }
+
+ /**
+ * Return the number of rows write.
+ */
+ public long getRowWritten() {
+ return rowWritten;
+ }
+
+ /**
+ * Increment the number of rows read.
+ */
+ public void incrRowReaded() {
+ rowReaded++;
+ }
+
+ /**
+ * Increment the number of rows skipped.
+ */
+ public void incrRowSkipped() {
+ rowSkipped++;
+ }
+
+ /**
+ * Increment the number of rows filtered.
+ */
+ public void incrRowFiltered() {
+ rowFiltered++;
+ }
+
+ /**
+ * Increment the number of rows parsed.
+ */
+ public void incrRowParsed() {
+ rowParsed++;
+ }
+
+ /**
+ * Increment the number of rows write.
+ */
+ public void incrRowWritten() {
+ rowWritten++;
+ }
+
+
+ public ToStringHelper toStringHelper(Object self) {
+ return MoreObjects.toStringHelper(self)
+ .add("entryName", entryName)
+ .add("elapsedTime", getElapsedTime())
+ .add("rowReaded", rowReaded)
+ .add("rowSkipped", rowSkipped)
+ .add("rowFiltered", rowFiltered)
+ .add("rowParsed", rowParsed)
+ .add("rowWritten", rowWritten);
+ }
+
+ @Override
+ public String toString() {
+ return toStringHelper(this).toString();
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/logging/AppenderBase.java b/services/billing-aws/src/main/java/com/epam/datalab/logging/AppenderBase.java
new file mode 100644
index 0000000..b3f7d97
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/logging/AppenderBase.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.logging;
+
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.OutputStreamAppender;
+import com.epam.datalab.exceptions.InitializationException;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.MoreObjects.ToStringHelper;
+
+import java.util.TimeZone;
+
+/**
+ * Abstract class provides base configuration for the log appenders.
+ */
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+public abstract class AppenderBase {
+
+ /**
+ * Log format pattern.
+ */
+ private final String logFormatPattern = "%-5p [%d{ISO8601," + TimeZone.getDefault().getID() + "}] %c: %m%n%rEx";
+
+ /**
+ * Perform configure of appender.
+ *
+ * @param context the context of logger.
+ */
+ public abstract void configure(LoggerContext context) throws InitializationException;
+
+ /**
+ * Perform the base configure of appender.
+ *
+ * @param context the context of logger.
+ * @param appenderName the name of appender.
+ * @param appender the class instance of appender.
+ */
+ public void configure(LoggerContext context, String appenderName, OutputStreamAppender<ILoggingEvent> appender) {
+ PatternLayoutEncoder encoder = new PatternLayoutEncoder();
+ encoder.setPattern(logFormatPattern);
+ encoder.setContext(context);
+ encoder.start();
+
+ appender.setContext(context);
+ appender.setName(appenderName);
+ appender.setEncoder(encoder);
+ appender.start();
+
+ Logger logger = context.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME);
+ logger.addAppender(appender);
+ logger.setAdditive(true);
+ }
+
+ /**
+ * Return the name of type for appender.
+ */
+ @JsonIgnore
+ public String getType() {
+ Class<? extends AppenderBase> clazz = this.getClass();
+ return (clazz.isAnnotationPresent(JsonTypeName.class) ?
+ clazz.getAnnotation(JsonTypeName.class).value() : clazz.getName());
+ }
+
+
+ /**
+ * Returns a string representation of the object.
+ *
+ * @param self the object to generate the string for (typically this), used only for its class name.
+ */
+ public ToStringHelper toStringHelper(Object self) {
+ return MoreObjects.toStringHelper(self)
+ .add("type", getType());
+ }
+
+ @Override
+ public String toString() {
+ return toStringHelper(this)
+ .toString();
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/logging/AppenderConsole.java b/services/billing-aws/src/main/java/com/epam/datalab/logging/AppenderConsole.java
new file mode 100644
index 0000000..0987bbf
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/logging/AppenderConsole.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.logging;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.ConsoleAppender;
+import com.epam.datalab.exceptions.InitializationException;
+import com.fasterxml.jackson.annotation.JsonClassDescription;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+/**
+ * Console appender for logging.
+ */
+@JsonTypeName("console")
+@JsonClassDescription(
+ "Console log appender.\n" +
+ "Output log data to console. Does not have any properties.\n" +
+ " - type: console"
+)
+public class AppenderConsole extends AppenderBase {
+
+ @Override
+ public void configure(LoggerContext context) throws InitializationException {
+ super.configure(context, "console-appender", new ConsoleAppender<ILoggingEvent>());
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/logging/AppenderFile.java b/services/billing-aws/src/main/java/com/epam/datalab/logging/AppenderFile.java
new file mode 100644
index 0000000..4ab53ad
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/logging/AppenderFile.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.logging;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.CoreConstants;
+import ch.qos.logback.core.FileAppender;
+import ch.qos.logback.core.rolling.DefaultTimeBasedFileNamingAndTriggeringPolicy;
+import ch.qos.logback.core.rolling.RollingFileAppender;
+import ch.qos.logback.core.rolling.TimeBasedFileNamingAndTriggeringPolicy;
+import ch.qos.logback.core.rolling.TimeBasedRollingPolicy;
+import com.epam.datalab.exceptions.InitializationException;
+import com.fasterxml.jackson.annotation.JsonClassDescription;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.MoreObjects.ToStringHelper;
+
+import javax.validation.Valid;
+import javax.validation.constraints.NotNull;
+
+/**
+ * File appender for logging. Support rolling files and archiving.
+ */
+@JsonTypeName("file")
+@JsonClassDescription(
+ "File log appender.\n" +
+ "Output log data to the file, if property archive is set to true then rolling\n" +
+ "mode is enabled. If archivedLogFilenamePattern ends with .gz or .zip extension\n" +
+ "then old log file will be compressed.\n" +
+ " - type: file\n" +
+ " currentLogFilename: <[path/]filename.log> - pattern for log file naming.\n" +
+ " [archive: <true | false>] - rolling log files or none.\n" +
+ " [archivedLogFilenamePattern: <[path/]filename-%d{yyyy-MM-dd}.log[.gz | .zip]>]\n" +
+ " - pattern for naming the archive log\n" +
+ " files.\n" +
+ " [archivedFileCount: <number_of_days>] - number of archive log file history."
+)
+public class AppenderFile extends AppenderBase {
+
+ /**
+ * The name of current log file.
+ */
+ @Valid
+ @NotNull
+ @JsonProperty
+ private String currentLogFilename;
+
+ /**
+ * Flag for archive of old files.
+ */
+ @Valid
+ @JsonProperty
+ private boolean archive = false;
+
+ /**
+ * Pattern for naming archive files. The compression mode depending on last
+ * letters of the fileNamePatternStr. Patterns ending with .gz imply GZIP
+ * compression, endings with '.zip' imply ZIP compression. Otherwise and by
+ * default, there is no compression.
+ */
+ @Valid
+ @JsonProperty
+ private String archivedLogFilenamePattern;
+
+ /**
+ * The maximum number of archive files to keep..
+ */
+ @Valid
+ @JsonProperty
+ private int archivedFileCount = CoreConstants.UNBOUND_HISTORY;
+
+
+ /**
+ * Return the name of current log file.
+ */
+ public String getCurrentLogFilename() {
+ return currentLogFilename;
+ }
+
+ /**
+ * Set the name of current log file.
+ */
+ public void setCurrentLogFilename(String currentLogFilename) {
+ this.currentLogFilename = currentLogFilename;
+ }
+
+ /**
+ * Return the flag for archive of old files.
+ */
+ public boolean getArchive() {
+ return archive;
+ }
+
+ /**
+ * Set the flag for archive of old files.
+ */
+ public void setArchive(boolean archive) {
+ this.archive = archive;
+ }
+
+ /**
+ * Return the pattern for naming archive files.
+ */
+ public String getArchivedLogFilenamePattern() {
+ return archivedLogFilenamePattern;
+ }
+
+ /**
+ * Set pattern for naming archive files. The compression mode depending on last
+ * letters of the fileNamePatternStr. Patterns ending with .gz imply GZIP
+ * compression, endings with '.zip' imply ZIP compression. Otherwise and by
+ * default, there is no compression.
+ * For example,
+ * /logs/application-%d{yyyy-MM-dd}.log.gz
+ */
+ public void setArchivedLogFilenamePattern(String archivedLogFilenamePattern) {
+ this.archivedLogFilenamePattern = archivedLogFilenamePattern;
+ }
+
+ /**
+ * Return the maximum number of archive files to keep..
+ */
+ public int getArchivedFileCount() {
+ return archivedFileCount;
+ }
+
+ /**
+ * Set the maximum number of archive files to keep..
+ */
+ public void setArchivedFileCount(int archivedFileCount) {
+ this.archivedFileCount = archivedFileCount;
+ }
+
+
+ @Override
+ public void configure(LoggerContext context) throws InitializationException {
+ if (currentLogFilename == null || currentLogFilename.trim().isEmpty()) {
+ throw new InitializationException("Configuration property logging.appenders.currentLogFilename cannot be null.");
+ }
+ super.configure(context, "file-appender", (archive ? getRollingFileAppender(context) : getFileAppender()));
+ }
+
+ /**
+ * Create and return synchronous the file appender.
+ */
+ private FileAppender<ILoggingEvent> getFileAppender() {
+ FileAppender<ILoggingEvent> appender = new FileAppender<>();
+ appender.setFile(currentLogFilename);
+ appender.setAppend(true);
+ return appender;
+ }
+
+ /**
+ * Create and return synchronous the rolling file appender.
+ *
+ * @param context the context of logger.
+ */
+ private RollingFileAppender<ILoggingEvent> getRollingFileAppender(LoggerContext context) throws InitializationException {
+ if (archivedLogFilenamePattern == null || archivedLogFilenamePattern.trim().isEmpty()) {
+ throw new InitializationException("Configuration property logging.appenders.archivedLogFilenamePattern cannot be null.");
+ }
+ RollingFileAppender<ILoggingEvent> appender = new RollingFileAppender<>();
+ appender.setFile(currentLogFilename);
+ appender.setAppend(true);
+
+ TimeBasedFileNamingAndTriggeringPolicy<ILoggingEvent> triggerPolicy = new DefaultTimeBasedFileNamingAndTriggeringPolicy<>();
+ triggerPolicy.setContext(context);
+
+ TimeBasedRollingPolicy<ILoggingEvent> rollPolicy = new TimeBasedRollingPolicy<>();
+ rollPolicy.setContext(context);
+ rollPolicy.setParent(appender);
+ rollPolicy.setFileNamePattern(archivedLogFilenamePattern);
+ rollPolicy.setMaxHistory(archivedFileCount);
+ rollPolicy.setTimeBasedFileNamingAndTriggeringPolicy(triggerPolicy);
+ rollPolicy.start();
+ appender.setRollingPolicy(rollPolicy);
+
+ return appender;
+ }
+
+
+ @Override
+ public ToStringHelper toStringHelper(Object self) {
+ return super.toStringHelper(self)
+ .add("currentLogFilename", currentLogFilename)
+ .add("archive", archive)
+ .add("archivedLogFilenamePattern", archivedLogFilenamePattern)
+ .add("archivedFileCount", archivedFileCount);
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/module/AdapterConsole.java b/services/billing-aws/src/main/java/com/epam/datalab/module/AdapterConsole.java
new file mode 100644
index 0000000..bca08b5
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/module/AdapterConsole.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.module;
+
+import com.epam.datalab.core.AdapterBase;
+import com.epam.datalab.core.parser.CommonFormat;
+import com.epam.datalab.exceptions.AdapterException;
+import com.epam.datalab.model.aws.ReportLine;
+import com.fasterxml.jackson.annotation.JsonClassDescription;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import org.bson.Document;
+
+import java.util.List;
+
+/**
+ * The adapter for console output.
+ */
+@JsonTypeName(ModuleName.ADAPTER_CONSOLE)
+@JsonClassDescription(
+ "Console adapter.\n" +
+ "Output data to console. Can be used for AdapterOut only.\n" +
+ " - type: " + ModuleName.ADAPTER_CONSOLE + "\n" +
+ " [writeHeader: <true | false>] - write header of data to the adapterOut."
+)
+public class AdapterConsole extends AdapterBase {
+
+ /**
+ * Default constructor for deserialization.
+ */
+ public AdapterConsole() {
+ }
+
+ /**
+ * Instantiate adapter for reading or writing.
+ *
+ * @param mode the mode of adapter.
+ */
+ public AdapterConsole(Mode mode) {
+ super(mode);
+ }
+
+
+ @Override
+ public void open() throws AdapterException {
+ if (getMode() != Mode.WRITE) {
+ throw new AdapterException("Mode of " + getType() + " adapter may be " + Mode.WRITE + " only.");
+ }
+ }
+
+ @Override
+ public void close() throws AdapterException {
+ // Nothing to do
+ }
+
+ @Override
+ public String getEntryName() {
+ return "console";
+ }
+
+ @Override
+ public String readLine() throws AdapterException {
+ throw new AdapterException("Unimplemented method called.");
+ }
+
+ @Override
+ public void writeHeader(List<String> header) {
+ System.out.println(CommonFormat.rowToString(header));
+ }
+
+ @Override
+ public Document writeRow(ReportLine row) {
+ System.out.println(CommonFormat.rowToString(row));
+ return null;
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/module/AdapterFile.java b/services/billing-aws/src/main/java/com/epam/datalab/module/AdapterFile.java
new file mode 100644
index 0000000..02af157
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/module/AdapterFile.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.module;
+
+import com.epam.datalab.core.AdapterBase;
+import com.epam.datalab.core.parser.CommonFormat;
+import com.epam.datalab.exceptions.AdapterException;
+import com.epam.datalab.model.aws.ReportLine;
+import com.fasterxml.jackson.annotation.JsonClassDescription;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.MoreObjects.ToStringHelper;
+import org.bson.Document;
+
+import javax.validation.constraints.NotNull;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * The adapter for file system.
+ */
+@JsonTypeName(ModuleName.ADAPTER_FILE)
+@JsonClassDescription(
+ "File adapter.\n" +
+ "Read source or write converted data to the file.\n" +
+ " - type: " + ModuleName.ADAPTER_FILE + "\n" +
+ " [writeHeader: <true | false>] - write header of data to the adapterOut.\n" +
+ " file: <filename> - the name of file."
+)
+public class AdapterFile extends AdapterBase {
+
+ /**
+ * The name of file.
+ */
+ @NotNull
+ @JsonProperty
+ private String file;
+ /**
+ * Reader for adapter.
+ */
+ @JsonIgnore
+ private BufferedReader reader;
+ /**
+ * Writer for adapter.
+ */
+ @JsonIgnore
+ private BufferedWriter writer;
+
+ /**
+ * Return the name of file.
+ */
+ public String getFile() {
+ return file;
+ }
+
+ /**
+ * Set the name of file.
+ */
+ public void setFile(String file) {
+ this.file = file;
+ }
+
+ @Override
+ public void open() throws AdapterException {
+ try {
+ if (getMode() == Mode.READ) {
+ reader = new BufferedReader(new FileReader(file));
+ } else if (getMode() == Mode.WRITE) {
+ writer = new BufferedWriter(new FileWriter(file));
+ } else {
+ throw new AdapterException("Mode of adapter unknown or not defined. Set mode to " + Mode.READ + " or " + Mode.WRITE + ".");
+ }
+ } catch (Exception e) {
+ throw new AdapterException("Cannot open file " + file + ". " + e.getLocalizedMessage(), e);
+ }
+ }
+
+ @Override
+ public void close() throws AdapterException {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ throw new AdapterException("Cannot close file " + file + ". " + e.getLocalizedMessage(), e);
+ } finally {
+ reader = null;
+ }
+ }
+
+ if (writer != null) {
+ try {
+ writer.close();
+ } catch (IOException e) {
+ throw new AdapterException("Cannot close file " + file + ". " + e.getLocalizedMessage(), e);
+ } finally {
+ writer = null;
+ }
+ }
+ }
+
+ @Override
+ public String getEntryName() {
+ return getFile();
+ }
+
+ @Override
+ public String readLine() throws AdapterException {
+ try {
+ return reader.readLine();
+ } catch (IOException e) {
+ throw new AdapterException("Cannot read file " + file + ". " + e.getLocalizedMessage(), e);
+ }
+ }
+
+ @Override
+ public void writeHeader(List<String> header) throws AdapterException {
+ try {
+ writer.write(CommonFormat.rowToString(header));
+ writer.write(System.lineSeparator());
+ } catch (IOException e) {
+ throw new AdapterException("Cannot write file " + file + ". " + e.getLocalizedMessage(), e);
+ }
+ }
+
+ @Override
+ public Document writeRow(ReportLine row) throws AdapterException {
+ try {
+ writer.write(CommonFormat.rowToString(row));
+ writer.write(System.lineSeparator());
+ } catch (IOException e) {
+ throw new AdapterException("Cannot write file " + file + ". " + e.getLocalizedMessage(), e);
+ }
+ return null;
+ }
+
+
+ @Override
+ public ToStringHelper toStringHelper(Object self) {
+ return super.toStringHelper(self)
+ .add("file", file);
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/module/ModuleName.java b/services/billing-aws/src/main/java/com/epam/datalab/module/ModuleName.java
new file mode 100644
index 0000000..a72bb06
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/module/ModuleName.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.module;
+
+/**
+ * Names of billing tool modules.
+ */
+public class ModuleName {
+ public static final String ADAPTER_CONSOLE = "console";
+ public static final String ADAPTER_AGG_CONSOLE = "aggConsole";
+ public static final String ADAPTER_FILE = "file";
+ public static final String ADAPTER_S3_FILE = "s3file";
+ public static final String ADAPTER_MONGO_DATALAB = "mongodatalab";
+ public static final String PARSER_CSV = "csv";
+ public static final String FILTER_AWS = "aws";
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/module/ParserCsv.java b/services/billing-aws/src/main/java/com/epam/datalab/module/ParserCsv.java
new file mode 100644
index 0000000..7a4bc2b
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/module/ParserCsv.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.module;
+
+import com.epam.datalab.core.parser.ParserByLine;
+import com.epam.datalab.exceptions.AdapterException;
+import com.epam.datalab.exceptions.InitializationException;
+import com.epam.datalab.exceptions.ParseException;
+import com.fasterxml.jackson.annotation.JsonClassDescription;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.MoreObjects.ToStringHelper;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.validation.constraints.NotNull;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Parse CSV format to common CSV format.
+ */
+@JsonTypeName(ModuleName.PARSER_CSV)
+@JsonClassDescription(
+ "CSV parser.\n" +
+ "Parse source CSV format to common billing report.\n" +
+ " - type: " + ModuleName.PARSER_CSV + "\n" +
+ " [dataFile: <filename>] - the file name to store working data of parser.]\n" +
+ " [columnStartDate: <column_name>] - the name of source column with date of data.]\n" +
+ " [columnMapping: >-\n" +
+ " <targetColumn1=sourceColumnX;targetColumn2=sourceColumnY; ...;\n" +
+ " tags=sourceColumnK,...,sourceColumnN>]\n" +
+ " - columns mapping to target from source columns.\n" +
+ " Know target columns: datalab_id, user,\n" +
+ " usage_date, product, usage_type, usage, cost,\n" +
+ " currency_code, resource_id, tags.\n" +
+ " [whereCondition: >-\n" +
+ " <(source_columnX > 0.0 || source_columnY == 'string') &&\n" +
+ " source_columnZ != 2016>]\n" +
+ " - where condition for filtering the source data,\n" +
+ " see http://commons.apache.org/proper/commons-jexl/reference/syntax.html#Operators\n" +
+ " for detais.\n" +
+ " [aggregate: <none | month | day>] - how to aggregate the data.\n" +
+ " [headerLineNo: <number>] - the number of header line in source data.\n" +
+ " [skipLines: <numbber>] - the number of line which will be skipped\n" +
+ " (include header).\n" +
+ " [fieldSeparator: <char>] - char for separate field names and values.\n" +
+ " [fieldTerminator: <char>] - char for terminate field names and values.\n" +
+ " [escapeChar: <char>] - escape char.\n" +
+ " [decimalSeparator: <char>] - char for decimal sign.\n" +
+ " [groupingSeparator: <char>] - char for thousands separator.\n"
+)
+public class ParserCsv extends ParserByLine {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ParserCsv.class);
+
+ /**
+ * Character for separate field names and values.
+ */
+ public static final char FIELD_SEPARATOR_DEFAULT = ',';
+
+ /**
+ * Character for termination field names and values.
+ */
+ public static final char FIELD_DELIMITER_DEFAULT = '"';
+
+ /**
+ * Escape character.
+ */
+ public static final char ESCAPE_CHAR_DEFAULT = '\\';
+
+
+ /**
+ * Character for separate field names and values.
+ */
+ @NotNull
+ @JsonProperty
+ private char fieldSeparator = FIELD_SEPARATOR_DEFAULT;
+
+ /**
+ * Character for termination field names and values.
+ */
+ @NotNull
+ @JsonProperty
+ private char fieldTerminator = FIELD_DELIMITER_DEFAULT;
+
+ /**
+ * Escape character.
+ */
+ @NotNull
+ @JsonProperty
+ private char escapeChar = ESCAPE_CHAR_DEFAULT;
+
+ /**
+ * The number of line that contain the header of data.
+ */
+ @JsonProperty
+ private int headerLineNo = 0;
+
+ /**
+ * The number of line which will be skipped (include header).
+ */
+ @JsonProperty
+ private int skipLines = 0;
+
+
+ /**
+ * Return the character for separate field names and values.
+ */
+ public char getFieldSeparator() {
+ return fieldSeparator;
+ }
+
+ /**
+ * Set the character for separate field names and values.
+ */
+ public void setFieldSeparator(char fieldSeparator) {
+ this.fieldSeparator = fieldSeparator;
+ }
+
+ /**
+ * Return the character for termination field names and values.
+ */
+ public char getFieldTerminator() {
+ return fieldTerminator;
+ }
+
+ /**
+ * Set the character for termination field names and values.
+ */
+ public void setFieldTerminator(char fieldTerminator) {
+ this.fieldTerminator = fieldTerminator;
+ }
+
+ /**
+ * Return the escape character.
+ */
+ public char getEscapeChar() {
+ return escapeChar;
+ }
+
+ /**
+ * Set the escape character.
+ */
+ public void setEscapeChar(char escapeChar) {
+ this.escapeChar = escapeChar;
+ }
+
+ /**
+ * Return the number of line that contain the header of data.
+ */
+ public int getHeaderLineNo() {
+ return headerLineNo;
+ }
+
+ /**
+ * Set the number of line that contain the header of data.
+ */
+ public void setHeaderLineNo(int headerLineNo) {
+ this.headerLineNo = headerLineNo;
+ }
+
+ /**
+ * Return the number of line which will be skipped (include header).
+ */
+ public int getSkipLines() {
+ return skipLines;
+ }
+
+ /**
+ * Set the number of line which will be skipped (include header).
+ */
+ public void setSkipLines(int skipLines) {
+ this.skipLines = skipLines;
+ }
+
+
+ @Override
+ public void initialize() throws InitializationException {
+ }
+
+ @Override
+ public List<String> parseHeader() throws AdapterException, ParseException {
+ String line = null;
+ List<String> header = null;
+
+ if (headerLineNo > 0) {
+ while (getCurrentStatistics().getRowReaded() < headerLineNo) {
+ if ((line = getNextRow()) == null) {
+ return null;
+ }
+ getCurrentStatistics().incrRowSkipped();
+ }
+ header = parseRow(line);
+ }
+
+ while (getCurrentStatistics().getRowReaded() < skipLines) {
+ if (getNextRow() == null) {
+ break;
+ }
+ getCurrentStatistics().incrRowSkipped();
+ }
+
+ return header;
+ }
+
+
+ /**
+ * Construct the exception.
+ *
+ * @param message the error message.
+ * @param pos the position in the parsed line.
+ * @param sourceLine the parsed line.
+ * @return ParseException
+ */
+ private ParseException getParseException(String message, int pos, String sourceLine) {
+ String s = String.format("%s at pos %d in line: ", message, pos);
+ LOGGER.error(s + sourceLine);
+ LOGGER.error(StringUtils.repeat(' ', s.length() + pos - 1) + '^');
+ return new ParseException(s + sourceLine);
+ }
+
+ @Override
+ public List<String> parseRow(String line) throws ParseException {
+ int realPos = 0;
+ int pos = 0;
+ boolean isDelimiter = false;
+ StringBuilder sb = new StringBuilder(line);
+ List<String> row = new ArrayList<>();
+
+ while (pos < sb.length()) {
+ char c = sb.charAt(pos);
+ if (c == escapeChar) {
+ realPos++;
+ pos++;
+ if (pos == sb.length()) {
+ throw getParseException("Invalid escape char", realPos, line);
+ }
+ sb.delete(pos - 1, pos);
+ realPos++;
+ } else if (c == fieldTerminator) {
+ realPos++;
+ if (isDelimiter) {
+ realPos++;
+ pos++;
+ if (pos == sb.length()) {
+ sb.delete(pos - 1, pos);
+ break;
+ }
+ if (sb.charAt(pos) == fieldSeparator) {
+ row.add(sb.substring(0, pos - 1));
+ sb.delete(0, pos + 1);
+ pos = 0;
+ isDelimiter = false;
+ continue;
+ }
+ throw getParseException("Invalid field delimiter", realPos, line);
+ }
+
+ if (pos != 0) {
+ throw getParseException("Unterminated field", realPos, line);
+ }
+ sb.delete(0, 1);
+ isDelimiter = true;
+ continue;
+ } else if (c == fieldSeparator) {
+ realPos++;
+ if (isDelimiter) {
+ pos++;
+ continue;
+ }
+ row.add(sb.substring(0, pos));
+ sb.delete(0, pos + 1);
+ pos = 0;
+ } else {
+ realPos++;
+ pos++;
+ }
+ }
+ row.add(sb.toString());
+
+ return row;
+ }
+
+
+ @Override
+ public ToStringHelper toStringHelper(Object self) {
+ return super.toStringHelper(self)
+ .add("fieldSeparator", fieldSeparator)
+ .add("fieldTerminator", fieldTerminator)
+ .add("escapeChar", escapeChar)
+ .add("headerLineNo", headerLineNo)
+ .add("skipLines", skipLines);
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/module/aws/AdapterS3File.java b/services/billing-aws/src/main/java/com/epam/datalab/module/aws/AdapterS3File.java
new file mode 100644
index 0000000..5bd7680
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/module/aws/AdapterS3File.java
@@ -0,0 +1,388 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.module.aws;
+
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+import com.epam.datalab.core.AdapterBase;
+import com.epam.datalab.exceptions.AdapterException;
+import com.epam.datalab.model.aws.ReportLine;
+import com.epam.datalab.module.ModuleName;
+import com.fasterxml.jackson.annotation.JsonClassDescription;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.MoreObjects.ToStringHelper;
+import org.bson.Document;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.validation.constraints.NotNull;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Date;
+import java.util.List;
+
+/**
+ * The adapter for S3 file system of Amazon.
+ */
+@JsonTypeName(ModuleName.ADAPTER_S3_FILE)
+@JsonClassDescription(
+ "Amazon S3 file system adapter.\n" +
+ "Read source or write converted data to the file in Amazon S3 bucket.\n" +
+ " - type: " + ModuleName.ADAPTER_S3_FILE + "\n" +
+ " [writeHeader: <true | false>] - write header of data to the adapterOut.\n" +
+ " bucket: <bucketname> - the name of S3 bucket.\n" +
+ " path: <path> - the path to the report or empty if used the root folder.\n" +
+ " accountId: <AWS account number> - the account number, see for details\n" +
+ " \"Detailed billing report with resources and tags\"\n" +
+ " http://docs.aws.amazon" +
+ ".com/awsaccountbilling/latest/aboutv2/billing-reports.html#detailed-report-with-resources-tags\n" +
+ " [accessKeyId: <string>] - Amazon access key ID.\n" +
+ " [secretAccessKey: <string>] - Amazon secret access key."
+)
+public class AdapterS3File extends AdapterBase {
+ /**
+ * Name of key for the last loaded file.
+ */
+ public static final String DATA_KEY_LAST_LOADED_FILE = "AdapterS3File_lastLoadedFile";
+ /**
+ * Name of key for the modification date of loaded file.
+ */
+ public static final String DATA_KEY_LAST_MODIFICATION_DATE = "AdapterS3File_lastModifyDate";
+ private static final Logger LOGGER = LoggerFactory.getLogger(AdapterS3File.class);
+ private static final String CANNOT_READ_FILE_FORMAT = "Cannot read file %s. %s";
+ private static final String DELIMITER = "/";
+
+ /**
+ * The name of bucket.
+ */
+ @NotNull
+ @JsonProperty
+ private String bucket;
+
+ /**
+ * The path to report.
+ */
+ @JsonProperty
+ private String path;
+
+ /**
+ * AWS account number.
+ */
+ @NotNull
+ @JsonProperty
+ private String accountId;
+
+ /**
+ * Access key ID for Amazon Web Services.
+ */
+ @JsonProperty
+ private String accessKeyId;
+
+ /**
+ * Secret key for Amazon Web Services.
+ */
+ @JsonProperty
+ private String secretAccessKey;
+
+ @JsonProperty
+ private boolean awsJobEnabled;
+ /**
+ * List of report files for loading.
+ */
+ @JsonIgnore
+ private List<String> filelist = null;
+ /**
+ * Index of current report file.
+ */
+ @JsonIgnore
+ private int currentFileIndex = -1;
+ /**
+ * Index of current report file.
+ */
+ @JsonIgnore
+ private String entryName = null;
+ /**
+ * Amazon S3 client.
+ */
+ @JsonIgnore
+ private AmazonS3 clientS3 = null;
+ /**
+ * Amazon S3 client.
+ */
+ @JsonIgnore
+ private Date lastModificationDate = null;
+ /**
+ * File input stream.
+ */
+ @JsonIgnore
+ private InputStream fileInputStream = null;
+ /**
+ * Reader for adapter.
+ */
+ @JsonIgnore
+ private BufferedReader reader = null;
+
+ /**
+ * Return the name of bucket.
+ */
+ public String getBucket() {
+ return bucket;
+ }
+
+ /**
+ * Set the name of bucket.
+ */
+ public void setBucket(String bucket) {
+ this.bucket = bucket;
+ }
+
+ /**
+ * Return the path to report.
+ */
+ public String getPath() {
+ return path;
+ }
+
+ /**
+ * Set the path to report.
+ */
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ /**
+ * Return the AWS account number.
+ */
+ public String getAccountId() {
+ return accountId;
+ }
+
+ /**
+ * Set the AWS account number.
+ */
+ public void setAccountId(String accountId) {
+ this.accountId = accountId;
+ }
+
+ /**
+ * Return the access key ID for Amazon Web Services.
+ */
+ public String getAccessKeyId() {
+ return this.accessKeyId;
+ }
+
+ /**
+ * Set the access key ID for Amazon Web Services.
+ */
+ public void setAccessKeyId(String accessKeyId) {
+ this.accessKeyId = accessKeyId;
+ }
+
+ /**
+ * Return the secret key for Amazon Web Services.
+ */
+ public String getSecretAccessKey() {
+ return this.secretAccessKey;
+ }
+
+ /**
+ * Set the secret key for Amazon Web Services.
+ */
+ public void setSecretAccessKey(String secretAccessKey) {
+ this.secretAccessKey = secretAccessKey;
+ }
+
+ @Override
+ public void open() throws AdapterException {
+ LOGGER.debug("Adapter S3 will be opened for {}", getMode());
+ if (getMode() == Mode.READ) {
+ setLastModificationDate();
+ clientS3 = getAmazonClient();
+ S3FileList s3files = new S3FileList(awsJobEnabled, bucket, getModuleData());
+ filelist = s3files.getFiles(clientS3);
+ currentFileIndex = (filelist.isEmpty() ? -1 : 0);
+ fileInputStream = null;
+ reader = null;
+ entryName = null;
+ openNextEntry();
+ LOGGER.debug("Adapter S3 has been opened");
+ } else if (getMode() == Mode.WRITE) {
+ throw new AdapterException("Unsupported mode " + Mode.WRITE + ".");
+ } else {
+ throw new AdapterException("Mode of adapter unknown or not defined. Set mode to " + Mode.READ + ".");
+ }
+ }
+
+ @Override
+ public boolean hasMultyEntry() {
+ return true;
+ }
+
+ @Override
+ public boolean openNextEntry() throws AdapterException {
+ String filename = getCurrentFileName();
+ if (filename == null) {
+ if (filelist.isEmpty()) {
+ final String reportPath = path == null ? bucket : bucket + DELIMITER + path;
+ LOGGER.debug("New report files in bucket folder {} not found", reportPath);
+ }
+ return false;
+ }
+ entryName = filename;
+ LOGGER.debug("Open a next entry in file {}", filename);
+ reader = new BufferedReader(new InputStreamReader(getFileStream()));
+ try {
+ getModuleData().setId(filename);
+ getModuleData().setModificationDate(lastModificationDate);
+ getModuleData().set(DATA_KEY_LAST_LOADED_FILE, filename);
+ getModuleData().set(DATA_KEY_LAST_MODIFICATION_DATE, lastModificationDate);
+ getModuleData().store();
+ } catch (Exception e) {
+ throw new AdapterException(e.getLocalizedMessage(), e);
+ }
+ currentFileIndex++;
+ return false;
+ }
+
+ @Override
+ public boolean hasEntryData() {
+ return (reader != null);
+ }
+
+ @Override
+ public void close() throws AdapterException {
+ closeFile(getCurrentFileName());
+ }
+
+ @Override
+ public String getEntryName() {
+ return entryName;
+ }
+
+ @Override
+ public String readLine() throws AdapterException {
+ try {
+ return reader.readLine();
+ } catch (IOException e) {
+ throw new AdapterException(String.format(CANNOT_READ_FILE_FORMAT, getCurrentFileName(), e
+ .getLocalizedMessage()), e);
+ }
+ }
+
+ @Override
+ public void writeHeader(List<String> header) throws AdapterException {
+ throw new AdapterException("Unimplemented method.");
+ }
+
+ @Override
+ public Document writeRow(ReportLine row) throws AdapterException {
+ throw new AdapterException("Unimplemented method.");
+ }
+
+ /**
+ * Return the current file name.
+ */
+ public String getCurrentFileName() {
+ return (filelist == null || currentFileIndex < 0 || currentFileIndex >= filelist.size() ? null : filelist.get
+ (currentFileIndex));
+ }
+
+ /**
+ * Creates and returns the Amazon client, as well as checks bucket existence.
+ *
+ * @throws AdapterException
+ */
+ private AmazonS3 getAmazonClient() throws AdapterException {
+ AmazonS3 s3 = (accessKeyId == null ?
+ new AmazonS3Client() :
+ new AmazonS3Client(new BasicAWSCredentials(accessKeyId, secretAccessKey)));
+
+ if (!s3.doesBucketExist(bucket)) {
+ throw new AdapterException("Bucket \"" + bucket + "\" does not exist.");
+ }
+
+ return s3;
+ }
+
+ /**
+ * Open the source file and return reader.
+ *
+ * @throws AdapterException
+ */
+ private InputStream getFileStream() throws AdapterException {
+ try {
+ GetObjectRequest request = new GetObjectRequest(bucket, getCurrentFileName());
+ S3Object object = clientS3.getObject(request);
+ lastModificationDate = object.getObjectMetadata().getLastModified();
+ return object.getObjectContent();
+ } catch (Exception e) {
+ throw new AdapterException("Cannot open file " + bucket + DELIMITER + getCurrentFileName() + ". " + e
+ .getLocalizedMessage(), e);
+ }
+ }
+
+ /**
+ * Return the modification date of loaded file.
+ *
+ * @throws AdapterException
+ */
+ private void setLastModificationDate() throws AdapterException {
+ try {
+ lastModificationDate = getModuleData().getDate(DATA_KEY_LAST_MODIFICATION_DATE);
+ } catch (Exception e) {
+ throw new AdapterException("Cannot get the last modification date for report. " + e.getLocalizedMessage(),
+ e);
+ }
+ }
+
+ /**
+ * Close a zip file.
+ *
+ * @param filename file name.
+ * @throws AdapterException
+ */
+ private void closeFile(String filename) throws AdapterException {
+ if (fileInputStream != null) {
+ try {
+ fileInputStream.close();
+ } catch (IOException e) {
+ throw new AdapterException("Cannot close file " + filename + ". " + e.getLocalizedMessage(), e);
+ }
+ fileInputStream = null;
+ }
+ }
+
+ @Override
+ public ToStringHelper toStringHelper(Object self) {
+ return super.toStringHelper(self)
+ .add("bucket", bucket)
+ .add("path", path)
+ .add("accountId", accountId)
+ .add("accessKeyId", "***")
+ .add("secretAccessKey", "***");
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/datalab/module/aws/FilterAWS.java b/services/billing-aws/src/main/java/com/epam/datalab/module/aws/FilterAWS.java
new file mode 100644
index 0000000..71154b5
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/datalab/module/aws/FilterAWS.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.datalab.module.aws;
+
+import com.epam.datalab.core.FilterBase;
+import com.epam.datalab.exceptions.InitializationException;
+import com.epam.datalab.model.aws.ReportLine;
+import com.epam.datalab.module.ModuleName;
+import com.fasterxml.jackson.annotation.JsonClassDescription;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.MoreObjects.ToStringHelper;
+
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+/**
+ * Filter and transform the line of AWS detailed billing reports.
+ */
+@JsonTypeName(ModuleName.FILTER_AWS)
+@JsonClassDescription(
+ "Amazon Web Services detailed billing reports filter.\n" +
+ "Filter report data and select line item only. Set column projectCode and\n" +
+ "currencyCode to user values.\n" +
+ " - type: " + ModuleName.FILTER_AWS + "\n" +
+ " [currencyCode: <string>] - user value for currencyCode column.\n" +
+ " [columnDatalabTag: <string>] - name of column tag of DataLab resource id.\n" +
+ " [serviceBaseName: <string>] - DataLab's service base name."
... 145663 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@dlab.apache.org
For additional commands, e-mail: commits-help@dlab.apache.org