You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fineract.apache.org by ar...@apache.org on 2023/02/06 18:53:08 UTC
[fineract] branch develop updated: FINERACT-1707 - S3 export and dependent service - [x] S3 export function - [x] list all available export type services - [x] refactor export service - [x] unit test for export name generator - [x] s3 integration test - [x] localstack integration for testing
This is an automated email from the ASF dual-hosted git repository.
arnold pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/fineract.git
The following commit(s) were added to refs/heads/develop by this push:
new d8fd8088b FINERACT-1707 - S3 export and dependent service - [x] S3 export function - [x] list all available export type services - [x] refactor export service - [x] unit test for export name generator - [x] s3 integration test - [x] localstack integration for testing
d8fd8088b is described below
commit d8fd8088b06f827a4b84080213ae0e25fe46835e
Author: Janos Haber <ja...@finesolution.hu>
AuthorDate: Thu Jan 26 22:01:06 2023 +0100
FINERACT-1707 - S3 export and dependent service
- [x] S3 export function
- [x] list all available export type services
- [x] refactor export service
- [x] unit test for export name generator
- [x] s3 integration test
- [x] localstack integration for testing
---
.github/workflows/build-mariadb.yml | 27 +++++
.github/workflows/build-mysql.yml | 27 +++++
.github/workflows/build-postgresql.yml | 27 +++++
.../domain/ConfigurationDomainService.java | 2 +
.../domain/ConfigurationDomainServiceJpa.java | 7 ++
.../core/config/FineractProperties.java | 24 +++++
.../service/StreamUtil.java} | 27 ++---
.../dataqueries/api/RunreportsApiResource.java | 29 +++++-
.../api/RunreportsApiResourceSwagger.java | 1 +
.../ReportExportType.java} | 28 ++----
.../service/DatatableExportTargetParameter.java | 2 +-
.../service/DatatableReportingProcessService.java | 110 +++++++--------------
.../CsvDatatableReportExportServiceImpl.java | 51 ++++++++++
.../service/export/DatatableExportUtil.java | 97 ++++++++++++++++++
.../export/DatatableReportExportService.java} | 26 ++---
.../export/Header.java} | 29 ++----
.../export/JsonDatatableReportExportService.java | 71 +++++++++++++
.../export/PdfDatatableReportExportService.java | 52 ++++++++++
.../export/ResponseHolder.java} | 33 ++++---
.../export/S3DatatableReportExportServiceImpl.java | 71 +++++++++++++
.../report/service/ReportingProcessService.java | 4 +
.../fineract/infrastructure/s3/AmazonS3Config.java | 65 ++++++++++++
.../infrastructure/s3/AmazonS3ConfigCondition.java | 45 +++++++++
.../LocalstackS3ClientCustomizer.java} | 36 +++----
.../S3ClientCustomizer.java} | 23 +----
.../src/main/resources/application.properties | 4 +
.../db/changelog/tenant/changelog-tenant.xml | 1 +
...0_add_report_export_s3_folder_configuration.xml | 37 +++++++
.../service/DatatableExportUtilTest.java | 97 ++++++++++++++++++
.../src/test/resources/application-test.properties | 2 +
.../apache/fineract/integrationtests/CIOnly.java | 31 ++----
.../integrationtests/client/ReportExportTest.java | 59 +++++++++++
.../integrationtests/client/ReportsTest.java | 9 ++
.../common/GlobalConfigurationHelper.java | 18 +++-
34 files changed, 948 insertions(+), 224 deletions(-)
diff --git a/.github/workflows/build-mariadb.yml b/.github/workflows/build-mariadb.yml
index e216973b7..eb5254e5f 100644
--- a/.github/workflows/build-mariadb.yml
+++ b/.github/workflows/build-mariadb.yml
@@ -52,12 +52,39 @@ jobs:
run: |
./gradlew --no-daemon -q createDB -PdbName=fineract_tenants
./gradlew --no-daemon -q createDB -PdbName=fineract_default
+ - name: Start LocalStack
+ env:
+ AWS_ENDPOINT_URL: http://localhost:4566
+ AWS_ACCESS_KEY_ID: localstack
+ AWS_SECRET_ACCESS_KEY: localstack
+ AWS_REGION: us-east-1
+ run: |
+ echo "Update python pyopenssl"
+ pip install --upgrade pyopenssl
+ echo "Install localstack"
+ pip install localstack awscli-local[ver1] # install LocalStack cli and awslocal
+ docker pull localstack/localstack # Make sure to pull the latest version of the image
+ localstack start -d # Start LocalStack in the background
+
+ echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
+ localstack wait -t 30 # to become ready before timing out
+ echo "Startup complete"
+ echo "Create fineract S3 bucket"
+ awslocal s3api create-bucket --bucket fineract-reports
+ echo "LocalStack initialization complete"
- name: Install additional software
run: |
sudo apt-get update
sudo apt-get install ghostscript graphviz -y
- name: Build & Test
+ env:
+ AWS_ENDPOINT_URL: http://localhost:4566
+ AWS_ACCESS_KEY_ID: localstack
+ AWS_SECRET_ACCESS_KEY: localstack
+ AWS_REGION: us-east-1
+ FINERACT_REPORT_EXPORT_S3_ENABLED: true
+ FINERACT_REPORT_EXPORT_S3_BUCKET_NAME: fineract-reports
run: |
./gradlew --no-daemon --console=plain build test --fail-fast -x doc -x :twofactor-tests:test -x :oauth2-test:test
./gradlew --no-daemon --console=plain :twofactor-tests:test --fail-fast
diff --git a/.github/workflows/build-mysql.yml b/.github/workflows/build-mysql.yml
index 4bb0d0635..ca7bf02d8 100644
--- a/.github/workflows/build-mysql.yml
+++ b/.github/workflows/build-mysql.yml
@@ -52,12 +52,39 @@ jobs:
run: |
./gradlew --no-daemon -q createMySQLDB -PdbName=fineract_tenants
./gradlew --no-daemon -q createMySQLDB -PdbName=fineract_default
+ - name: Start LocalStack
+ env:
+ AWS_ENDPOINT_URL: http://localhost:4566
+ AWS_ACCESS_KEY_ID: localstack
+ AWS_SECRET_ACCESS_KEY: localstack
+ AWS_REGION: us-east-1
+ run: |
+ echo "Update python pyopenssl"
+ pip install --upgrade pyopenssl
+ echo "Install localstack"
+ pip install localstack awscli-local[ver1] # install LocalStack cli and awslocal
+ docker pull localstack/localstack # Make sure to pull the latest version of the image
+ localstack start -d # Start LocalStack in the background
+
+ echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
+ localstack wait -t 30 # to become ready before timing out
+ echo "Startup complete"
+ echo "Create fineract S3 bucket"
+ awslocal s3api create-bucket --bucket fineract-reports
+ echo "LocalStack initialization complete"
- name: Install additional software
run: |
sudo apt-get update
sudo apt-get install ghostscript graphviz -y
- name: Build & Test
+ env:
+ AWS_ENDPOINT_URL: http://localhost:4566
+ AWS_ACCESS_KEY_ID: localstack
+ AWS_SECRET_ACCESS_KEY: localstack
+ AWS_REGION: us-east-1
+ FINERACT_REPORT_EXPORT_S3_ENABLED: true
+ FINERACT_REPORT_EXPORT_S3_BUCKET_NAME: fineract-reports
run: |
./gradlew --no-daemon --console=plain build test --fail-fast -x doc -x :twofactor-tests:test -x :oauth2-test:test -PdbType=mysql
./gradlew --no-daemon --console=plain :twofactor-tests:test --fail-fast -PdbType=mysql
diff --git a/.github/workflows/build-postgresql.yml b/.github/workflows/build-postgresql.yml
index 83382569b..3e861f7e6 100644
--- a/.github/workflows/build-postgresql.yml
+++ b/.github/workflows/build-postgresql.yml
@@ -53,12 +53,39 @@ jobs:
run: |
./gradlew --no-daemon -q createPGDB -PdbName=fineract_tenants
./gradlew --no-daemon -q createPGDB -PdbName=fineract_default
+ - name: Start LocalStack
+ env:
+ AWS_ENDPOINT_URL: http://localhost:4566
+ AWS_ACCESS_KEY_ID: localstack
+ AWS_SECRET_ACCESS_KEY: localstack
+ AWS_REGION: us-east-1
+ run: |
+ echo "Update python pyopenssl"
+ pip install --upgrade pyopenssl
+ echo "Install localstack"
+ pip install localstack awscli-local[ver1] # install LocalStack cli and awslocal
+ docker pull localstack/localstack # Make sure to pull the latest version of the image
+ localstack start -d # Start LocalStack in the background
+
+ echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
+ localstack wait -t 30 # to become ready before timing out
+ echo "Startup complete"
+ echo "Create fineract S3 bucket"
+ awslocal s3api create-bucket --bucket fineract-reports
+ echo "LocalStack initialization complete"
- name: Install additional software
run: |
sudo apt-get update
sudo apt-get install ghostscript graphviz -y
- name: Build & Test
+ env:
+ AWS_ENDPOINT_URL: http://localhost:4566
+ AWS_ACCESS_KEY_ID: localstack
+ AWS_SECRET_ACCESS_KEY: localstack
+ AWS_REGION: us-east-1
+ FINERACT_REPORT_EXPORT_S3_ENABLED: true
+ FINERACT_REPORT_EXPORT_S3_BUCKET_NAME: fineract-reports
run: |
./gradlew --no-daemon --console=plain build test --fail-fast -x doc -x :twofactor-tests:test -x :oauth2-test:test -PdbType=postgresql
./gradlew --no-daemon --console=plain :twofactor-tests:test --fail-fast -PdbType=postgresql
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java
index 168c1553e..a7b2cb580 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainService.java
@@ -134,4 +134,6 @@ public interface ConfigurationDomainService {
boolean isCOBBulkEventEnabled();
Long retrieveExternalEventBatchSize();
+
+ String retrieveReportExportS3FolderName();
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java
index c34ab9f9d..5088b3c38 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/configuration/domain/ConfigurationDomainServiceJpa.java
@@ -50,6 +50,8 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService
private static final String ENABLE_COB_BULK_EVENT = "enable-cob-bulk-event";
private static final String EXTERNAL_EVENT_BATCH_SIZE = "external-event-batch-size";
+ private static final String REPORT_EXPORT_S3_FOLDER_NAME = "report-export-s3-folder-name";
+
private final PermissionRepository permissionRepository;
private final GlobalConfigurationRepositoryWrapper globalConfigurationRepository;
private final PlatformCacheRepository cacheTypeRepository;
@@ -512,4 +514,9 @@ public class ConfigurationDomainServiceJpa implements ConfigurationDomainService
return property.getValue();
}
+ @Override
+ public String retrieveReportExportS3FolderName() {
+ final GlobalConfigurationPropertyData property = getGlobalConfigurationPropertyData(REPORT_EXPORT_S3_FOLDER_NAME);
+ return property.getStringValue();
+ }
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
index 99b50c5fd..85fbef4b7 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
@@ -48,6 +48,8 @@ public class FineractProperties {
private FineractContentProperties content;
+ private FineractReportProperties report;
+
@Getter
@Setter
public static class FineractTenantProperties {
@@ -201,4 +203,26 @@ public class FineractProperties {
private String accessKey;
private String secretKey;
}
+
+ @Getter
+ @Setter
+ public static class FineractReportProperties {
+
+ private FineractExportProperties export;
+ }
+
+ @Getter
+ @Setter
+ public static class FineractExportProperties {
+
+ private FineractExportS3Properties s3;
+ }
+
+ @Getter
+ @Setter
+ public static class FineractExportS3Properties {
+
+ private String bucketName;
+ private Boolean enabled;
+ }
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/StreamUtil.java
similarity index 54%
copy from fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
copy to fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/StreamUtil.java
index f91ab04e0..725a99d82 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/service/StreamUtil.java
@@ -16,26 +16,19 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.fineract.infrastructure.dataqueries.api;
+package org.apache.fineract.infrastructure.core.service;
-import io.swagger.v3.oas.annotations.media.Schema;
-import java.util.List;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetRowData;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
-/**
- * Created by sanyam on 5/8/17. Fixed ;) by Michael Vorburger.ch on 2020/11/21.
- */
-final class RunreportsApiResourceSwagger {
-
- private RunreportsApiResourceSwagger() {}
-
- @Schema
- public static final class RunReportsResponse {
+public final class StreamUtil {
- private RunReportsResponse() {}
+ private StreamUtil() {}
- public List<ResultsetColumnHeaderData> columnHeaders;
- public List<ResultsetRowData> data;
+ public static <A, B> Collector<A, ?, B> foldLeft(final B init, final BiFunction<? super B, ? super A, ? extends B> f) {
+ return Collectors.collectingAndThen(Collectors.reducing(Function.<B>identity(), a -> b -> f.apply(b, a), Function::andThen),
+ endo -> endo.apply(init));
}
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResource.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResource.java
index 0718e8b14..ab859a25c 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResource.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResource.java
@@ -20,6 +20,7 @@ package org.apache.fineract.infrastructure.dataqueries.api;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.ArraySchema;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
@@ -39,6 +40,7 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import org.apache.fineract.infrastructure.core.api.ApiParameterHelper;
import org.apache.fineract.infrastructure.core.exception.PlatformServiceUnavailableException;
+import org.apache.fineract.infrastructure.dataqueries.data.ReportExportType;
import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
import org.apache.fineract.infrastructure.report.provider.ReportingProcessServiceProvider;
import org.apache.fineract.infrastructure.report.service.ReportingProcessService;
@@ -70,6 +72,29 @@ public class RunreportsApiResource {
this.reportingProcessServiceProvider = reportingProcessServiceProvider;
}
+ @GET
+ @Path("/availableExports/{reportName}")
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ @Operation(summary = "Return all available export types for the specific report", description = "Returns the list of all available export types.")
+ @ApiResponses({
+ @ApiResponse(responseCode = "200", description = "", content = @Content(array = @ArraySchema(schema = @Schema(implementation = ReportExportType.class)))) })
+ public Response retrieveAllAvailableExports(@PathParam("reportName") @Parameter(description = "reportName") final String reportName,
+ @Context final UriInfo uriInfo,
+ @DefaultValue("false") @QueryParam(IS_SELF_SERVICE_USER_REPORT_PARAMETER) @Parameter(description = IS_SELF_SERVICE_USER_REPORT_PARAMETER) final boolean isSelfServiceUserReport) {
+ MultivaluedMap<String, String> queryParams = new MultivaluedStringMap();
+ queryParams.putAll(uriInfo.getQueryParameters());
+
+ final boolean parameterType = ApiParameterHelper.parameterType(queryParams);
+ String reportType = readExtraDataAndReportingService.getReportType(reportName, isSelfServiceUserReport, parameterType);
+ ReportingProcessService reportingProcessService = reportingProcessServiceProvider.findReportingProcessService(reportType);
+ if (reportingProcessService == null) {
+ throw new PlatformServiceUnavailableException("err.msg.report.service.implementation.missing",
+ ReportingProcessServiceProvider.SERVICE_MISSING + reportType, reportType);
+ }
+ return Response.ok().entity(reportingProcessService.getAvailableExportTargets()).build();
+ }
+
@GET
@Path("{reportName}")
@Consumes({ MediaType.APPLICATION_JSON })
@@ -108,8 +133,8 @@ public class RunreportsApiResource {
// Pass through isSelfServiceUserReport so that ReportingProcessService implementations can use it
queryParams.putSingle(IS_SELF_SERVICE_USER_REPORT_PARAMETER, Boolean.toString(isSelfServiceUserReport));
- String reportType = this.readExtraDataAndReportingService.getReportType(reportName, isSelfServiceUserReport, parameterType);
- ReportingProcessService reportingProcessService = this.reportingProcessServiceProvider.findReportingProcessService(reportType);
+ String reportType = readExtraDataAndReportingService.getReportType(reportName, isSelfServiceUserReport, parameterType);
+ ReportingProcessService reportingProcessService = reportingProcessServiceProvider.findReportingProcessService(reportType);
if (reportingProcessService == null) {
throw new PlatformServiceUnavailableException("err.msg.report.service.implementation.missing",
ReportingProcessServiceProvider.SERVICE_MISSING + reportType, reportType);
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
index f91ab04e0..712ef97bc 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
@@ -38,4 +38,5 @@ final class RunreportsApiResourceSwagger {
public List<ResultsetColumnHeaderData> columnHeaders;
public List<ResultsetRowData> data;
}
+
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ReportExportType.java
similarity index 53%
copy from fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
copy to fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ReportExportType.java
index f91ab04e0..10ea4daba 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ReportExportType.java
@@ -16,26 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.fineract.infrastructure.dataqueries.api;
+package org.apache.fineract.infrastructure.dataqueries.data;
-import io.swagger.v3.oas.annotations.media.Schema;
-import java.util.List;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetRowData;
+import java.io.Serializable;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
-/**
- * Created by sanyam on 5/8/17. Fixed ;) by Michael Vorburger.ch on 2020/11/21.
- */
-final class RunreportsApiResourceSwagger {
-
- private RunreportsApiResourceSwagger() {}
-
- @Schema
- public static final class RunReportsResponse {
-
- private RunReportsResponse() {}
+@Data
+@RequiredArgsConstructor
+public class ReportExportType implements Serializable {
- public List<ResultsetColumnHeaderData> columnHeaders;
- public List<ResultsetRowData> data;
- }
+ private final String key;
+ private final String queryParameter;
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java
index e596b182e..4c5d22a80 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java
@@ -34,7 +34,7 @@ public enum DatatableExportTargetParameter {
return this.value;
}
- public static DatatableExportTargetParameter checkTarget(final MultivaluedMap<String, String> queryParams) {
+ public static DatatableExportTargetParameter resolverExportTarget(final MultivaluedMap<String, String> queryParams) {
for (DatatableExportTargetParameter parameter : DatatableExportTargetParameter.values()) {
String parameterName = parameter.getValue();
if (queryParams.getFirst(parameterName) != null) {
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java
index b5f2e584b..85176b567 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java
@@ -18,105 +18,71 @@
*/
package org.apache.fineract.infrastructure.dataqueries.service;
-import java.io.File;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import javax.ws.rs.core.MediaType;
+import java.util.Optional;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.ResponseBuilder;
-import javax.ws.rs.core.StreamingOutput;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
import org.apache.fineract.infrastructure.core.api.ApiParameterHelper;
-import org.apache.fineract.infrastructure.core.serialization.ToApiJsonSerializer;
+import org.apache.fineract.infrastructure.core.service.StreamUtil;
import org.apache.fineract.infrastructure.dataqueries.api.RunreportsApiResource;
-import org.apache.fineract.infrastructure.dataqueries.data.GenericResultsetData;
-import org.apache.fineract.infrastructure.dataqueries.data.ReportData;
+import org.apache.fineract.infrastructure.dataqueries.data.ReportExportType;
+import org.apache.fineract.infrastructure.dataqueries.service.export.DatatableReportExportService;
+import org.apache.fineract.infrastructure.dataqueries.service.export.ResponseHolder;
import org.apache.fineract.infrastructure.report.annotation.ReportService;
import org.apache.fineract.infrastructure.report.service.ReportingProcessService;
-import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
@ReportService(type = { "Table", "Chart", "SMS" })
+@RequiredArgsConstructor
+@Slf4j
public class DatatableReportingProcessService implements ReportingProcessService {
- private final ReadReportingService readExtraDataAndReportingService;
- private final ToApiJsonSerializer<ReportData> toApiJsonSerializer;
- private final GenericDataService genericDataService;
-
- @Autowired
- public DatatableReportingProcessService(final ReadReportingService readExtraDataAndReportingService,
- final GenericDataService genericDataService, final ToApiJsonSerializer<ReportData> toApiJsonSerializer) {
- this.readExtraDataAndReportingService = readExtraDataAndReportingService;
- this.toApiJsonSerializer = toApiJsonSerializer;
- this.genericDataService = genericDataService;
- }
+ private final List<DatatableReportExportService> exportServices;
@Override
public Response processRequest(String reportName, MultivaluedMap<String, String> queryParams) {
boolean isSelfServiceUserReport = Boolean.parseBoolean(
queryParams.getOrDefault(RunreportsApiResource.IS_SELF_SERVICE_USER_REPORT_PARAMETER, List.of("false")).get(0));
- DatatableExportTargetParameter exportMode = DatatableExportTargetParameter.checkTarget(queryParams);
+ DatatableExportTargetParameter exportMode = DatatableExportTargetParameter.resolverExportTarget(queryParams);
final String parameterTypeValue = ApiParameterHelper.parameterType(queryParams) ? "parameter" : "report";
final Map<String, String> reportParams = getReportParams(queryParams);
- return switch (exportMode) {
- case CSV -> exportCSV(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue);
- case PDF -> exportPDF(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue);
- case S3 -> exportS3(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue);
- default -> exportJSON(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue,
- exportMode == DatatableExportTargetParameter.PRETTY_JSON);
- };
- }
-
- private Response exportJSON(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
- boolean isSelfServiceUserReport, String parameterTypeValue, boolean prettyPrint) {
- final GenericResultsetData result = this.readExtraDataAndReportingService.retrieveGenericResultset(reportName, parameterTypeValue,
- reportParams, isSelfServiceUserReport);
-
- String json;
- final boolean genericResultSetIsPassed = ApiParameterHelper.genericResultSetPassed(queryParams);
- final boolean genericResultSet = ApiParameterHelper.genericResultSet(queryParams);
- if (genericResultSetIsPassed) {
- if (genericResultSet) {
- json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
- } else {
- json = this.genericDataService.generateJsonFromGenericResultsetData(result);
- }
- } else {
- json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
+ ResponseHolder response = findReportExportService(exportMode) //
+ .orElseThrow(() -> new IllegalArgumentException("Unsupported export target: " + exportMode)) //
+ .export(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue);
+ Response.ResponseBuilder builder = Response.status(response.status().getStatusCode());
+ if (StringUtils.isNotBlank(response.contentType())) {
+ builder = builder.type(response.contentType());
}
-
- return Response.ok().entity(json).type(MediaType.APPLICATION_JSON).build();
+ if (StringUtils.isNotBlank(response.fileName())) {
+ builder = builder.header("Content-Disposition", "attachment; filename=" + response.fileName());
+ }
+ if (response.entity() != null) {
+ builder = builder.entity(response.entity());
+ }
+ if (response.headers() != null && !response.headers().isEmpty()) {
+ builder = response.headers().stream().collect(StreamUtil.foldLeft(builder, (b, h) -> b.header(h.getKey(), h.getValue())));
+ }
+ return builder.build();
}
- private Response exportS3(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
- boolean isSelfServiceUserReport, String parameterTypeValue) {
- throw new UnsupportedOperationException("S3 export not supported for datatables");
+ @Override
+ public List<ReportExportType> getAvailableExportTargets() {
+ return Arrays //
+ .stream(DatatableExportTargetParameter.values()) //
+ .filter(target -> findReportExportService(target).isPresent()) //
+ .map(target -> new ReportExportType(target.name(), target.getValue())) //
+ .toList();
}
- private Response exportPDF(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
- boolean isSelfServiceUserReport, String parameterTypeValue) {
-
- final String pdfFileName = this.readExtraDataAndReportingService.retrieveReportPDF(reportName, parameterTypeValue, reportParams,
- isSelfServiceUserReport);
-
- final File file = new File(pdfFileName);
-
- final ResponseBuilder response = Response.ok(file);
- response.header("Content-Disposition", "attachment; filename=\"" + pdfFileName + "\"");
- response.header("content-Type", "application/pdf");
-
- return response.build();
+ private Optional<DatatableReportExportService> findReportExportService(DatatableExportTargetParameter target) {
+ return exportServices.stream().filter(service -> service.supports(target)).findFirst();
}
- private Response exportCSV(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
- boolean isSelfServiceUserReport, String parameterTypeValue) {
- final StreamingOutput result = this.readExtraDataAndReportingService.retrieveReportCSV(reportName, parameterTypeValue, reportParams,
- isSelfServiceUserReport);
-
- return Response.ok().entity(result).type("text/csv")
- .header("Content-Disposition", "attachment;filename=" + reportName.replaceAll(" ", "") + ".csv").build();
-
- }
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/CsvDatatableReportExportServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/CsvDatatableReportExportServiceImpl.java
new file mode 100644
index 000000000..bb5525eb8
--- /dev/null
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/CsvDatatableReportExportServiceImpl.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.dataqueries.service.export;
+
+import java.util.Map;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+import lombok.RequiredArgsConstructor;
+import org.apache.fineract.infrastructure.dataqueries.service.DatatableExportTargetParameter;
+import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
+import org.springframework.stereotype.Service;
+
+@Service
+@RequiredArgsConstructor
+public class CsvDatatableReportExportServiceImpl implements DatatableReportExportService {
+
+ private final ReadReportingService readExtraDataAndReportingService;
+
+ @Override
+ public ResponseHolder export(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
+ boolean isSelfServiceUserReport, String parameterTypeValue) {
+ final StreamingOutput result = this.readExtraDataAndReportingService.retrieveReportCSV(reportName, parameterTypeValue, reportParams,
+ isSelfServiceUserReport);
+ return new ResponseHolder(Response.Status.OK).contentType("text/csv")
+ .addHeader("Content-Disposition",
+ "attachment;filename=" + DatatableExportUtil.generatePlainExportFileName(255, "csv", reportName, reportParams))
+ .entity(result);
+ }
+
+ @Override
+ public boolean supports(DatatableExportTargetParameter exportType) {
+ return exportType == DatatableExportTargetParameter.CSV;
+ }
+}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/DatatableExportUtil.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/DatatableExportUtil.java
new file mode 100644
index 000000000..0a6b522c4
--- /dev/null
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/DatatableExportUtil.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.dataqueries.service.export;
+
+import java.time.format.DateTimeFormatter;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.fineract.infrastructure.core.service.DateUtils;
+import org.jetbrains.annotations.NotNull;
+
+public final class DatatableExportUtil {
+
+ private DatatableExportUtil() {}
+
+ public static String normalizeFolderName(String folderName) {
+ if (StringUtils.isBlank(folderName)) {
+ return "";
+ }
+ String contentNormalizer = folderName.trim() //
+ .replaceAll("[^a-zA-Z0-9!\\-_.'()$/]", "_") // replace special characters
+ .replaceAll("/+", "/") // replace multiply / with a single /
+ .replaceAll("^[./]+", ""); // remove leading . and /
+
+ return contentNormalizer.endsWith("/") ? contentNormalizer.substring(0, contentNormalizer.length() - 1) : contentNormalizer;
+ }
+
+ public static String generatePlainExportFileName(int maxLength, String extension, String reportName, Map<String, String> reportParams) {
+ exportBasicValidation(extension, reportName);
+ return generateReportFileName(maxLength, "", extension, reportName, reportParams);
+ }
+
+ private static void exportBasicValidation(String extension, String reportName) {
+ if (StringUtils.isBlank(extension)) {
+ throw new IllegalArgumentException("The extension is required");
+ }
+ if (StringUtils.isBlank(reportName)) {
+ throw new IllegalArgumentException("The report name is required");
+ }
+ }
+
+ public static String generateS3DatatableExportFileName(int maxLength, String folder, String extension, String reportName,
+ Map<String, String> reportParams) {
+ exportBasicValidation(extension, reportName);
+ if (maxLength < 30) {
+ throw new IllegalArgumentException("The maximum length must be greater than 30");
+ }
+ folder = normalizeFolderName(folder);
+ String reportFinalName = generateReportFileName(maxLength, folder, extension, reportName, reportParams);
+ if (StringUtils.isBlank(folder)) {
+ return reportFinalName;
+ } else {
+ return folder + "/" + reportFinalName;
+ }
+ }
+
+ @NotNull
+ private static String generateReportFileName(int maxLength, String folder, String extension, String reportName,
+ Map<String, String> reportParams) {
+ String extensionWithDot = extension.startsWith(".") ? extension : "." + extension;
+ String timestamp = "_" + DateUtils.getOffsetDateTimeOfTenant().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss"));
+
+ int reportMaximumFileName = maxLength - folder.length() - timestamp.length() - extensionWithDot.length() - 1;
+ if (reportMaximumFileName < 0) {
+ throw new IllegalArgumentException("The folder name is too long");
+ }
+ String normalizedFileName = reportName.trim().replaceAll("[^a-zA-Z0-9!\\-_.'()$]", "_");
+ if (reportParams != null) {
+ normalizedFileName += "(" + reportParams.entrySet().stream()
+ .map(entry -> extractReportParameterKey(entry.getKey()) + "_" + entry.getValue()).collect(Collectors.joining(";"))
+ + ")";
+ }
+ String reportFinalName = normalizedFileName.substring(0, Math.min(normalizedFileName.length(), reportMaximumFileName)) + timestamp
+ + extensionWithDot;
+ return reportFinalName;
+ }
+
+ private static String extractReportParameterKey(String key) {
+ return key.startsWith("${") && key.endsWith("}") ? key.substring(2, key.length() - 1) : key;
+ }
+}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/DatatableReportExportService.java
similarity index 54%
copy from fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
copy to fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/DatatableReportExportService.java
index f91ab04e0..4af32530a 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/DatatableReportExportService.java
@@ -16,26 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.fineract.infrastructure.dataqueries.api;
+package org.apache.fineract.infrastructure.dataqueries.service.export;
-import io.swagger.v3.oas.annotations.media.Schema;
-import java.util.List;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetRowData;
+import java.util.Map;
+import javax.ws.rs.core.MultivaluedMap;
+import org.apache.fineract.infrastructure.dataqueries.service.DatatableExportTargetParameter;
-/**
- * Created by sanyam on 5/8/17. Fixed ;) by Michael Vorburger.ch on 2020/11/21.
- */
-final class RunreportsApiResourceSwagger {
-
- private RunreportsApiResourceSwagger() {}
-
- @Schema
- public static final class RunReportsResponse {
+public interface DatatableReportExportService {
- private RunReportsResponse() {}
+ ResponseHolder export(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
+ boolean isSelfServiceUserReport, String parameterTypeValue);
- public List<ResultsetColumnHeaderData> columnHeaders;
- public List<ResultsetRowData> data;
- }
+ boolean supports(DatatableExportTargetParameter exportType);
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/Header.java
similarity index 53%
copy from fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
copy to fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/Header.java
index f91ab04e0..40032c933 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/Header.java
@@ -16,26 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.fineract.infrastructure.dataqueries.api;
+package org.apache.fineract.infrastructure.dataqueries.service.export;
-import io.swagger.v3.oas.annotations.media.Schema;
-import java.util.List;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetRowData;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
-/**
- * Created by sanyam on 5/8/17. Fixed ;) by Michael Vorburger.ch on 2020/11/21.
- */
-final class RunreportsApiResourceSwagger {
-
- private RunreportsApiResourceSwagger() {}
-
- @Schema
- public static final class RunReportsResponse {
-
- private RunReportsResponse() {}
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class Header {
- public List<ResultsetColumnHeaderData> columnHeaders;
- public List<ResultsetRowData> data;
- }
+ private String key;
+ private String value;
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/JsonDatatableReportExportService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/JsonDatatableReportExportService.java
new file mode 100644
index 000000000..a5eb1a266
--- /dev/null
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/JsonDatatableReportExportService.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.dataqueries.service.export;
+
+import java.util.Map;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
+import lombok.RequiredArgsConstructor;
+import org.apache.fineract.infrastructure.core.api.ApiParameterHelper;
+import org.apache.fineract.infrastructure.core.serialization.ToApiJsonSerializer;
+import org.apache.fineract.infrastructure.dataqueries.data.GenericResultsetData;
+import org.apache.fineract.infrastructure.dataqueries.data.ReportData;
+import org.apache.fineract.infrastructure.dataqueries.service.DatatableExportTargetParameter;
+import org.apache.fineract.infrastructure.dataqueries.service.GenericDataService;
+import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
+import org.springframework.stereotype.Service;
+
+@Service
+@RequiredArgsConstructor
+public class JsonDatatableReportExportService implements DatatableReportExportService {
+
+ private final ReadReportingService readExtraDataAndReportingService;
+ private final ToApiJsonSerializer<ReportData> toApiJsonSerializer;
+ private final GenericDataService genericDataService;
+
+ @Override
+ public ResponseHolder export(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
+ boolean isSelfServiceUserReport, String parameterTypeValue) {
+
+ final GenericResultsetData result = this.readExtraDataAndReportingService.retrieveGenericResultset(reportName, parameterTypeValue,
+ reportParams, isSelfServiceUserReport);
+ DatatableExportTargetParameter exportMode = DatatableExportTargetParameter.resolverExportTarget(queryParams);
+ boolean prettyPrint = exportMode == DatatableExportTargetParameter.PRETTY_JSON;
+ String json;
+ final boolean genericResultSetIsPassed = ApiParameterHelper.genericResultSetPassed(queryParams);
+ final boolean genericResultSet = ApiParameterHelper.genericResultSet(queryParams);
+ if (genericResultSetIsPassed) {
+ if (genericResultSet) {
+ json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
+ } else {
+ json = this.genericDataService.generateJsonFromGenericResultsetData(result);
+ }
+ } else {
+ json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
+ }
+ return new ResponseHolder(Response.Status.OK).entity(json).contentType(MediaType.APPLICATION_JSON);
+
+ }
+
+ @Override
+ public boolean supports(DatatableExportTargetParameter exportType) {
+ return exportType == DatatableExportTargetParameter.JSON || exportType == DatatableExportTargetParameter.PRETTY_JSON;
+ }
+}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/PdfDatatableReportExportService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/PdfDatatableReportExportService.java
new file mode 100644
index 000000000..d4e9a4a45
--- /dev/null
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/PdfDatatableReportExportService.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.dataqueries.service.export;
+
+import java.io.File;
+import java.util.Map;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
+import lombok.RequiredArgsConstructor;
+import org.apache.fineract.infrastructure.dataqueries.service.DatatableExportTargetParameter;
+import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
+import org.springframework.stereotype.Service;
+
+@Service
+@RequiredArgsConstructor
+public class PdfDatatableReportExportService implements DatatableReportExportService {
+
+ private final ReadReportingService readExtraDataAndReportingService;
+
+ @Override
+ public ResponseHolder export(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
+ boolean isSelfServiceUserReport, String parameterTypeValue) {
+ final String pdfFileName = this.readExtraDataAndReportingService.retrieveReportPDF(reportName, parameterTypeValue, reportParams,
+ isSelfServiceUserReport);
+
+ final File file = new File(pdfFileName);
+
+ return new ResponseHolder(Response.Status.OK).contentType("application/pdf")
+ .addHeader("Content-Disposition", "attachment; filename=\"" + pdfFileName + "\"").entity(file);
+ }
+
+ @Override
+ public boolean supports(DatatableExportTargetParameter exportType) {
+ return exportType == DatatableExportTargetParameter.PDF;
+ }
+}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/ResponseHolder.java
similarity index 56%
copy from fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
copy to fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/ResponseHolder.java
index f91ab04e0..5ed4a4796 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/ResponseHolder.java
@@ -16,26 +16,31 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.fineract.infrastructure.dataqueries.api;
+package org.apache.fineract.infrastructure.dataqueries.service.export;
-import io.swagger.v3.oas.annotations.media.Schema;
+import java.util.ArrayList;
import java.util.List;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetRowData;
+import javax.ws.rs.core.Response;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.Accessors;
-/**
- * Created by sanyam on 5/8/17. Fixed ;) by Michael Vorburger.ch on 2020/11/21.
- */
-final class RunreportsApiResourceSwagger {
+@Data
+@Accessors(fluent = true)
+@RequiredArgsConstructor
+public class ResponseHolder {
- private RunreportsApiResourceSwagger() {}
+ private String contentType;
+ private String fileName;
+ private final Response.Status status;
- @Schema
- public static final class RunReportsResponse {
+ private Object entity;
- private RunReportsResponse() {}
+ private List<Header> headers = new ArrayList<>();
- public List<ResultsetColumnHeaderData> columnHeaders;
- public List<ResultsetRowData> data;
+ public ResponseHolder addHeader(String key, String value) {
+ headers.add(new Header(key, value));
+ return this;
}
+
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/S3DatatableReportExportServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/S3DatatableReportExportServiceImpl.java
new file mode 100644
index 000000000..3cc509334
--- /dev/null
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/export/S3DatatableReportExportServiceImpl.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.dataqueries.service.export;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Map;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+import lombok.RequiredArgsConstructor;
+import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import org.apache.fineract.infrastructure.dataqueries.service.DatatableExportTargetParameter;
+import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+
+@RequiredArgsConstructor
+public class S3DatatableReportExportServiceImpl implements DatatableReportExportService {
+
+ public static final int AWS_S3_MAXIMUM_KEY_LENGTH = 1024;
+ private final ReadReportingService readExtraDataAndReportingService;
+
+ private final ConfigurationDomainService configurationDomainService;
+ private final S3Client s3Client;
+
+ private final FineractProperties properties;
+
+ @Override
+ public ResponseHolder export(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
+ boolean isSelfServiceUserReport, String parameterTypeValue) {
+ try {
+ StreamingOutput output = this.readExtraDataAndReportingService.retrieveReportCSV(reportName, parameterTypeValue, reportParams,
+ isSelfServiceUserReport);
+ try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
+ output.write(byteArrayOutputStream);
+ String folder = configurationDomainService.retrieveReportExportS3FolderName();
+ String filePath = DatatableExportUtil.generateS3DatatableExportFileName(AWS_S3_MAXIMUM_KEY_LENGTH, folder, "csv",
+ reportName, reportParams);
+ s3Client.putObject(
+ builder -> builder.bucket(properties.getReport().getExport().getS3().getBucketName()).key(filePath).build(),
+ RequestBody.fromBytes(byteArrayOutputStream.toByteArray()));
+ return new ResponseHolder(Response.Status.NO_CONTENT);
+ }
+ } catch (IOException e) {
+ throw new IllegalStateException("Error while exporting to S3", e);
+ }
+ }
+
+ @Override
+ public boolean supports(DatatableExportTargetParameter exportType) {
+ return DatatableExportTargetParameter.S3 == exportType;
+ }
+}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/report/service/ReportingProcessService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/report/service/ReportingProcessService.java
index 406157f54..e4fd348af 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/report/service/ReportingProcessService.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/report/service/ReportingProcessService.java
@@ -19,16 +19,20 @@
package org.apache.fineract.infrastructure.report.service;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
+import org.apache.fineract.infrastructure.dataqueries.data.ReportExportType;
import org.apache.fineract.infrastructure.security.utils.SQLInjectionValidator;
public interface ReportingProcessService {
Response processRequest(String reportName, MultivaluedMap<String, String> queryParams);
+ List<ReportExportType> getAvailableExportTargets();
+
default Map<String, String> getReportParams(final MultivaluedMap<String, String> queryParams) {
final Map<String, String> reportParams = new HashMap<>();
final Set<String> keys = queryParams.keySet();
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/AmazonS3Config.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/AmazonS3Config.java
new file mode 100644
index 000000000..1458c9391
--- /dev/null
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/AmazonS3Config.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.s3;
+
+import java.util.List;
+import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
+import org.apache.fineract.infrastructure.dataqueries.service.export.S3DatatableReportExportServiceImpl;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.context.annotation.Configuration;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.regions.providers.AwsRegionProvider;
+import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+
+@Configuration
+@Conditional(AmazonS3ConfigCondition.class)
+public class AmazonS3Config {
+
+ @Bean
+ public DefaultCredentialsProvider awsCredentialsProvider() {
+ return DefaultCredentialsProvider.create();
+ }
+
+ @Bean
+ public AwsRegionProvider awsRegionProvider() {
+ return DefaultAwsRegionProviderChain.builder().build();
+ }
+
+ @Bean("s3Client")
+ public S3Client s3Client(DefaultCredentialsProvider awsCredentialsProvider, AwsRegionProvider awsRegionProvider,
+ List<S3ClientCustomizer> customizers) {
+ S3ClientBuilder builder = S3Client.builder().credentialsProvider(awsCredentialsProvider).region(awsRegionProvider.getRegion());
+ customizers.forEach(customizer -> customizer.customize(builder));
+ return builder.build();
+ }
+
+ @Bean
+ @ConditionalOnBean(S3Client.class)
+ public S3DatatableReportExportServiceImpl s3DatatableReportExportServiceImpl(ReadReportingService reportServiceImpl,
+ ConfigurationDomainService configurationDomainService, S3Client s3Client, FineractProperties fineractProperties) {
+ return new S3DatatableReportExportServiceImpl(reportServiceImpl, configurationDomainService, s3Client, fineractProperties);
+ }
+
+}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/AmazonS3ConfigCondition.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/AmazonS3ConfigCondition.java
new file mode 100644
index 000000000..932b1bc94
--- /dev/null
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/AmazonS3ConfigCondition.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.s3;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.fineract.infrastructure.core.condition.PropertiesCondition;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
+
+public class AmazonS3ConfigCondition extends PropertiesCondition {
+
+ @Override
+ protected boolean matches(FineractProperties properties) {
+ FineractProperties.FineractExportS3Properties s3ReportExportProperties = properties.getReport().getExport().getS3();
+ return s3ReportExportProperties.getEnabled() && StringUtils.isNotBlank(s3ReportExportProperties.getBucketName())
+ && isAwsCredentialValid();
+ }
+
+ private boolean isAwsCredentialValid() {
+ try {
+ DefaultCredentialsProvider.create().resolveCredentials();
+ DefaultAwsRegionProviderChain.builder().build().getRegion();
+ return true;
+ } catch (Exception e) {
+ return false;
+ }
+ }
+}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/LocalstackS3ClientCustomizer.java
similarity index 50%
copy from fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
copy to fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/LocalstackS3ClientCustomizer.java
index f91ab04e0..6b764aa6a 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/LocalstackS3ClientCustomizer.java
@@ -16,26 +16,28 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.fineract.infrastructure.dataqueries.api;
+package org.apache.fineract.infrastructure.s3;
-import io.swagger.v3.oas.annotations.media.Schema;
-import java.util.List;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetRowData;
+import java.net.URI;
+import lombok.RequiredArgsConstructor;
+import org.apache.poi.util.StringUtil;
+import org.springframework.context.annotation.Profile;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Component;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
-/**
- * Created by sanyam on 5/8/17. Fixed ;) by Michael Vorburger.ch on 2020/11/21.
- */
-final class RunreportsApiResourceSwagger {
-
- private RunreportsApiResourceSwagger() {}
-
- @Schema
- public static final class RunReportsResponse {
+@Component
+@RequiredArgsConstructor
+@Profile("test")
+public class LocalstackS3ClientCustomizer implements S3ClientCustomizer {
- private RunReportsResponse() {}
+ private final Environment environment;
- public List<ResultsetColumnHeaderData> columnHeaders;
- public List<ResultsetRowData> data;
+ @Override
+ public void customize(S3ClientBuilder builder) {
+ String env = environment.getProperty("AWS_ENDPOINT_URL", "");
+ if (StringUtil.isNotBlank(env)) {
+ builder.endpointOverride(URI.create(env)).forcePathStyle(true);
+ }
}
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/S3ClientCustomizer.java
similarity index 53%
copy from fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
copy to fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/S3ClientCustomizer.java
index f91ab04e0..14821aa23 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/s3/S3ClientCustomizer.java
@@ -16,26 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.fineract.infrastructure.dataqueries.api;
+package org.apache.fineract.infrastructure.s3;
-import io.swagger.v3.oas.annotations.media.Schema;
-import java.util.List;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetRowData;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
-/**
- * Created by sanyam on 5/8/17. Fixed ;) by Michael Vorburger.ch on 2020/11/21.
- */
-final class RunreportsApiResourceSwagger {
-
- private RunreportsApiResourceSwagger() {}
-
- @Schema
- public static final class RunReportsResponse {
-
- private RunReportsResponse() {}
+public interface S3ClientCustomizer {
- public List<ResultsetColumnHeaderData> columnHeaders;
- public List<ResultsetRowData> data;
- }
+ void customize(S3ClientBuilder builder);
}
diff --git a/fineract-provider/src/main/resources/application.properties b/fineract-provider/src/main/resources/application.properties
index a322b44b4..c7f8078ab 100644
--- a/fineract-provider/src/main/resources/application.properties
+++ b/fineract-provider/src/main/resources/application.properties
@@ -86,6 +86,10 @@ fineract.content.s3.bucketName=${FINERACT_CONTENT_S3_BUCKET_NAME:}
fineract.content.s3.accessKey=${FINERACT_CONTENT_S3_ACCESS_KEY:}
fineract.content.s3.secretKey=${FINERACT_CONTENT_S3_SECRET_KEY:}
+
+fineract.report.export.s3.bucket=${FINERACT_REPORT_EXPORT_S3_BUCKET_NAME:}
+fineract.report.export.s3.enabled=${FINERACT_REPORT_EXPORT_S3_ENABLED:false}
+
# Logging pattern for the console
logging.pattern.console=${CONSOLE_LOG_PATTERN:%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(%replace([%X{correlationId}]){'\\[\\]', ''}) %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:%wEx}}
diff --git a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml
index 21ea1ef16..7786690da 100644
--- a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml
+++ b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml
@@ -109,4 +109,5 @@
<include file="parts/0087_update_dashboard_table_reports.xml" relativeToChangelogFile="true" />
<include file="parts/0088_drop_m_loan_transaction_version_column.xml" relativeToChangelogFile="true" />
<include file="parts/0089_add_update_loan_arrears_aging_business_step.xml" relativeToChangelogFile="true" />
+ <include file="parts/0090_add_report_export_s3_folder_configuration.xml" relativeToChangelogFile="true" />
</databaseChangeLog>
diff --git a/fineract-provider/src/main/resources/db/changelog/tenant/parts/0090_add_report_export_s3_folder_configuration.xml b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0090_add_report_export_s3_folder_configuration.xml
new file mode 100644
index 000000000..1050bfc44
--- /dev/null
+++ b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0090_add_report_export_s3_folder_configuration.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.3.xsd">
+ <changeSet author="fineract" id="1">
+ <insert tableName="c_configuration">
+ <column name="id" valueNumeric="54"/>
+ <column name="name" value="report-export-s3-folder-name"/>
+ <column name="value"/>
+ <column name="date_value"/>
+ <column name="string_value" value="reports"/>
+ <column name="enabled" valueBoolean="true"/>
+ <column name="is_trap_door" valueBoolean="false"/>
+ <column name="description"/>
+ </insert>
+ </changeSet>
+</databaseChangeLog>
diff --git a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportUtilTest.java b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportUtilTest.java
new file mode 100644
index 000000000..1ff9c2944
--- /dev/null
+++ b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportUtilTest.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.dataqueries.service;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.fineract.infrastructure.dataqueries.service.export.DatatableExportUtil;
+import org.junit.Assert;
+import org.junit.jupiter.api.Test;
+
+public class DatatableExportUtilTest {
+
+ @Test
+ public void emptyFolderTest() {
+ Assert.assertEquals("", DatatableExportUtil.normalizeFolderName(""));
+ Assert.assertEquals("", DatatableExportUtil.normalizeFolderName("/"));
+ Assert.assertEquals("", DatatableExportUtil.normalizeFolderName(null));
+ }
+
+ @Test
+ public void specialCharacterFolderTest() {
+ Assert.assertEquals("_", DatatableExportUtil.normalizeFolderName("Á"));
+ Assert.assertEquals("_", DatatableExportUtil.normalizeFolderName("="));
+ Assert.assertEquals("_", DatatableExportUtil.normalizeFolderName("\\"));
+ Assert.assertEquals("_", DatatableExportUtil.normalizeFolderName("@"));
+ }
+
+ @Test
+ public void normalizedFolderNameTest() {
+ Assert.assertEquals("$", DatatableExportUtil.normalizeFolderName("$"));
+ Assert.assertEquals("reports", DatatableExportUtil.normalizeFolderName("reports"));
+ Assert.assertEquals("reports", DatatableExportUtil.normalizeFolderName("reports/"));
+ Assert.assertEquals("reports", DatatableExportUtil.normalizeFolderName("/reports/"));
+ Assert.assertEquals("reports/content", DatatableExportUtil.normalizeFolderName("reports/content"));
+ Assert.assertEquals("reports/content", DatatableExportUtil.normalizeFolderName("reports/////content"));
+ }
+
+ @Test
+ public void generateDatatableExportFileNameSuccessTest() {
+ String reportName = "reportName";
+ Map<String, String> reportParams = Collections.synchronizedSortedMap(new TreeMap<>(Map.of("param1", "value1", "param2", "value2")));
+ String fileName = DatatableExportUtil.generateS3DatatableExportFileName(1024, "folder", "csv", reportName, reportParams);
+ Assert.assertTrue(fileName.matches("folder/reportName\\(param1_value1;param2_value2\\)_\\d{14}.csv"));
+ }
+
+ @Test
+ public void generateDatatableExportFileNameComplexTest() {
+ String reportName = "reportName";
+ Map<String, String> reportParams = Collections.synchronizedSortedMap(new TreeMap<>(Map.of("param1", "value1", "param2", "value2")));
+ Assert.assertTrue(DatatableExportUtil.generateS3DatatableExportFileName(1024, "folder///name///", "csv", reportName, reportParams)
+ .matches("folder/name/reportName\\(param1_value1;param2_value2\\)_\\d{14}.csv"));
+ IllegalArgumentException folderTooLongException = Assert.assertThrows(IllegalArgumentException.class, () -> {
+ DatatableExportUtil.generateS3DatatableExportFileName(30, "too_long_folder_name_test", "csv", reportName, reportParams);
+ });
+ Assert.assertEquals("The folder name is too long", folderTooLongException.getMessage());
+
+ IllegalArgumentException maximumLengthException = Assert.assertThrows(IllegalArgumentException.class, () -> {
+ DatatableExportUtil.generateS3DatatableExportFileName(29, "folder///name/", "csv", reportName, reportParams);
+ });
+ Assert.assertEquals("The maximum length must be greater than 30", maximumLengthException.getMessage());
+
+ IllegalArgumentException extensionRequired = Assert.assertThrows(IllegalArgumentException.class, () -> {
+ DatatableExportUtil.generateS3DatatableExportFileName(30, "too_long_folder_name_test", null, reportName, reportParams);
+ });
+ Assert.assertEquals("The extension is required", extensionRequired.getMessage());
+
+ IllegalArgumentException reportNameRequired = Assert.assertThrows(IllegalArgumentException.class, () -> {
+ DatatableExportUtil.generateS3DatatableExportFileName(30, "too_long_folder_name_test", "csv", null, reportParams);
+ });
+ Assert.assertEquals("The report name is required", reportNameRequired.getMessage());
+
+ Assert.assertTrue(DatatableExportUtil.generateS3DatatableExportFileName(1024, "folder///name/", ".csv", reportName, null)
+ .matches("folder/name/reportName_\\d{14}.csv"));
+
+ Assert.assertTrue(
+ DatatableExportUtil.generateS3DatatableExportFileName(1024, "folder///name/", "csv", "report/with/slash", reportParams)
+ .matches("folder/name/report_with_slash\\(param1_value1;param2_value2\\)_\\d{14}.csv"));
+ }
+
+}
diff --git a/fineract-provider/src/test/resources/application-test.properties b/fineract-provider/src/test/resources/application-test.properties
index 492a2aa65..3e2f22432 100644
--- a/fineract-provider/src/test/resources/application-test.properties
+++ b/fineract-provider/src/test/resources/application-test.properties
@@ -70,6 +70,8 @@ fineract.content.s3.enabled=false
fineract.content.s3.bucketName=
fineract.content.s3.accessKey=
fineract.content.s3.secretKey=
+fineract.report.export.s3.bucket=${FINERACT_REPORT_EXPORT_S3_BUCKET_NAME:}
+fineract.report.export.s3.enabled=${FINERACT_REPORT_EXPORT_S3_ENABLED:false}
management.health.jms.enabled=false
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/CIOnly.java
similarity index 53%
copy from fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
copy to integration-tests/src/test/java/org/apache/fineract/integrationtests/CIOnly.java
index f91ab04e0..47e3d6649 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/api/RunreportsApiResourceSwagger.java
+++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/CIOnly.java
@@ -16,26 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.fineract.infrastructure.dataqueries.api;
+package org.apache.fineract.integrationtests;
-import io.swagger.v3.oas.annotations.media.Schema;
-import java.util.List;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData;
-import org.apache.fineract.infrastructure.dataqueries.data.ResultsetRowData;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
-/**
- * Created by sanyam on 5/8/17. Fixed ;) by Michael Vorburger.ch on 2020/11/21.
- */
-final class RunreportsApiResourceSwagger {
-
- private RunreportsApiResourceSwagger() {}
-
- @Schema
- public static final class RunReportsResponse {
-
- private RunReportsResponse() {}
-
- public List<ResultsetColumnHeaderData> columnHeaders;
- public List<ResultsetRowData> data;
- }
-}
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.TYPE, ElementType.METHOD })
+@EnabledIfEnvironmentVariable(named = "GITHUB_ACTIONS", matches = "true")
+public @interface CIOnly {}
diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportExportTest.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportExportTest.java
new file mode 100644
index 000000000..3de4ddbc3
--- /dev/null
+++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportExportTest.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.integrationtests.client;
+
+import java.io.IOException;
+import java.util.Map;
+import okhttp3.MediaType;
+import okhttp3.ResponseBody;
+import org.apache.fineract.integrationtests.CIOnly;
+import org.apache.fineract.integrationtests.common.Utils;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import retrofit2.Response;
+
+/**
+ * Integration Test for /runreports/ API.
+ *
+ * @author Michael Vorburger.ch
+ */
+public class ReportExportTest extends IntegrationTest {
+
+ @BeforeEach
+ public void setup() {
+ Utils.initializeRESTAssured();
+ }
+
+ @Test
+ void runClientListingTableReportCSV() throws IOException {
+ Response<ResponseBody> result = okR(
+ fineract().reportsRun.runReportGetFile("Client Listing", Map.of("R_officeId", "1", "exportCSV", "true"), false));
+ assertThat(result.body().contentType()).isEqualTo(MediaType.parse("text/csv"));
+ assertThat(result.body().string()).contains("Office/Branch");
+ }
+
+ @Test
+ @CIOnly
+ void runClientListingTableReportS3() throws IOException {
+ Response<ResponseBody> result = okR(
+ fineract().reportsRun.runReportGetFile("Client Listing", Map.of("R_officeId", "1", "exportS3", "true"), false));
+ assertThat(result.code()).isEqualTo(204);
+ }
+
+}
diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportsTest.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportsTest.java
index 72bfcef1d..7c538c4f1 100644
--- a/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportsTest.java
+++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ReportsTest.java
@@ -27,6 +27,7 @@ import org.apache.fineract.integrationtests.common.Utils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+import retrofit2.Response;
/**
* Integration Test for /runreports/ API.
@@ -51,6 +52,14 @@ public class ReportsTest extends IntegrationTest {
.getColumnName()).isEqualTo("Office/Branch");
}
+ @Test
+ void runClientListingTableReportCSV() throws IOException {
+ Response<ResponseBody> result = okR(
+ fineract().reportsRun.runReportGetFile("Client Listing", Map.of("R_officeId", "1", "exportCSV", "true"), false));
+ assertThat(result.body().contentType()).isEqualTo(MediaType.parse("text/csv"));
+ assertThat(result.body().string()).contains("Office/Branch");
+ }
+
@Test // see FINERACT-1306
void runReportCategory() throws IOException {
// Using raw OkHttp instead of Retrofit API here, because /runreports/reportCategoryList returns JSON Array -
diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java
index a92e8246b..6ba54e4c7 100644
--- a/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java
+++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/common/GlobalConfigurationHelper.java
@@ -96,9 +96,9 @@ public class GlobalConfigurationHelper {
// If any other column is modified by the integration test suite in
// the future, it needs to be reset here.
final Integer configDefaultId = (Integer) defaultGlobalConfiguration.get("id");
- final Integer configDefaultValue = (Integer) defaultGlobalConfiguration.get("value");
+ final String configDefaultValue = String.valueOf(defaultGlobalConfiguration.get("value"));
- updateValueForGlobalConfiguration(requestSpec, responseSpec, configDefaultId.toString(), configDefaultValue.toString());
+ updateValueForGlobalConfiguration(requestSpec, responseSpec, configDefaultId.toString(), configDefaultValue);
updateEnabledFlagForGlobalConfiguration(requestSpec, responseSpec, configDefaultId.toString(),
(Boolean) defaultGlobalConfiguration.get("enabled"));
changedNo++;
@@ -119,9 +119,9 @@ public class GlobalConfigurationHelper {
ArrayList<HashMap> expectedGlobalConfigurations = getAllDefaultGlobalConfigurations();
ArrayList<HashMap> actualGlobalConfigurations = getAllGlobalConfigurations(requestSpec, responseSpec);
- // There are currently 48 global configurations.
- Assertions.assertEquals(48, expectedGlobalConfigurations.size());
- Assertions.assertEquals(48, actualGlobalConfigurations.size());
+ // There are currently 49 global configurations.
+ Assertions.assertEquals(49, expectedGlobalConfigurations.size());
+ Assertions.assertEquals(49, actualGlobalConfigurations.size());
for (int i = 0; i < expectedGlobalConfigurations.size(); i++) {
@@ -540,6 +540,14 @@ public class GlobalConfigurationHelper {
externalEventBatchSize.put("enabled", false);
externalEventBatchSize.put("trapDoor", false);
defaults.add(externalEventBatchSize);
+
+ HashMap<String, Object> reportExportS3FolderName = new HashMap<>();
+ reportExportS3FolderName.put("id", 54);
+ reportExportS3FolderName.put("name", "report-export-s3-folder-name");
+ reportExportS3FolderName.put("value", 0);
+ reportExportS3FolderName.put("enabled", false);
+ reportExportS3FolderName.put("trapDoor", false);
+ defaults.add(reportExportS3FolderName);
return defaults;
}