You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fineract.apache.org by ar...@apache.org on 2023/01/19 14:45:55 UTC
[fineract] branch develop updated: FINERACT-1707 - Export refactor [x] Refactor Datatable export [x] Fix CSV exceptions
This is an automated email from the ASF dual-hosted git repository.
arnold pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/fineract.git
The following commit(s) were added to refs/heads/develop by this push:
new e82c0cac5 FINERACT-1707 - Export refactor [x] Refactor Datatable export [x] Fix CSV exceptions
e82c0cac5 is described below
commit e82c0cac5222e354e001a65a90aaccee7b17b186
Author: Janos Haber <ja...@finesolution.hu>
AuthorDate: Tue Jan 17 23:02:13 2023 +0100
FINERACT-1707 - Export refactor
[x] Refactor Datatable export
[x] Fix CSV exceptions
---
.../groovy/org.apache.fineract.dependencies.gradle | 1 +
fineract-provider/dependencies.gradle | 1 +
.../core/api/ApiParameterHelper.java | 18 -----
.../service/DatatableExportTargetParameter.java | 48 ++++++++++++
.../service/DatatableReportingProcessService.java | 86 ++++++++++++----------
.../service/ReadReportingServiceImpl.java | 77 ++++---------------
6 files changed, 115 insertions(+), 116 deletions(-)
diff --git a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle
index a58531cff..4d53d1334 100644
--- a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle
+++ b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle
@@ -60,6 +60,7 @@ dependencyManagement {
dependency ('org.mnode.ical4j:ical4j:3.2.8') {
exclude 'com.sun.mail:javax.mail'
}
+ dependency 'org.apache.commons:commons-csv:1.9.0'
dependency 'org.quartz-scheduler:quartz:2.3.2'
dependency 'software.amazon.awssdk:bom:2.15.0'
dependency 'org.ehcache:ehcache:3.10.8'
diff --git a/fineract-provider/dependencies.gradle b/fineract-provider/dependencies.gradle
index c5d85003a..6ad81837a 100644
--- a/fineract-provider/dependencies.gradle
+++ b/fineract-provider/dependencies.gradle
@@ -121,6 +121,7 @@ dependencies {
exclude group: 'com.zaxxer', module: 'HikariCP-java7'
}
implementation platform('software.amazon.awssdk:bom')
+ implementation('org.apache.commons:commons-csv'){}
implementation ('software.amazon.awssdk:s3') {
}
implementation ('software.amazon.awssdk:auth') {
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/api/ApiParameterHelper.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/api/ApiParameterHelper.java
index b48d6c95b..89ce6c1d3 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/api/ApiParameterHelper.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/api/ApiParameterHelper.java
@@ -100,24 +100,6 @@ public final class ApiParameterHelper {
return locale;
}
- public static boolean exportCsv(final MultivaluedMap<String, String> queryParams) {
- boolean exportCsv = false;
- if (queryParams.getFirst("exportCSV") != null) {
- final String exportCsvValue = queryParams.getFirst("exportCSV");
- exportCsv = "true".equalsIgnoreCase(exportCsvValue);
- }
- return exportCsv;
- }
-
- public static boolean exportPdf(final MultivaluedMap<String, String> queryParams) {
- boolean exportPDF = false;
- if (queryParams.getFirst("exportPDF") != null) {
- final String exportPdfValue = queryParams.getFirst("exportPDF");
- exportPDF = "true".equalsIgnoreCase(exportPdfValue);
- }
- return exportPDF;
- }
-
public static boolean parameterType(final MultivaluedMap<String, String> queryParams) {
boolean parameterType = false;
if (queryParams.getFirst("parameterType") != null) {
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java
new file mode 100644
index 000000000..e596b182e
--- /dev/null
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableExportTargetParameter.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.dataqueries.service;
+
+import javax.ws.rs.core.MultivaluedMap;
+
+public enum DatatableExportTargetParameter {
+
+ CSV("exportCSV"), PDF("exportPDF"), S3("exportS3"), JSON("exportJSON"), PRETTY_JSON("pretty");
+
+ private final String value;
+
+ DatatableExportTargetParameter(final String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return this.value;
+ }
+
+ public static DatatableExportTargetParameter checkTarget(final MultivaluedMap<String, String> queryParams) {
+ for (DatatableExportTargetParameter parameter : DatatableExportTargetParameter.values()) {
+ String parameterName = parameter.getValue();
+ if (queryParams.getFirst(parameterName) != null) {
+ if ("true".equalsIgnoreCase(queryParams.getFirst(parameterName))) {
+ return parameter;
+ }
+ }
+ }
+ return JSON;
+ }
+}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java
index e7cb15653..b5f2e584b 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReportingProcessService.java
@@ -56,55 +56,67 @@ public class DatatableReportingProcessService implements ReportingProcessService
public Response processRequest(String reportName, MultivaluedMap<String, String> queryParams) {
boolean isSelfServiceUserReport = Boolean.parseBoolean(
queryParams.getOrDefault(RunreportsApiResource.IS_SELF_SERVICE_USER_REPORT_PARAMETER, List.of("false")).get(0));
- final boolean prettyPrint = ApiParameterHelper.prettyPrint(queryParams);
- final boolean exportCsv = ApiParameterHelper.exportCsv(queryParams);
- final boolean exportPdf = ApiParameterHelper.exportPdf(queryParams);
+
+ DatatableExportTargetParameter exportMode = DatatableExportTargetParameter.checkTarget(queryParams);
final String parameterTypeValue = ApiParameterHelper.parameterType(queryParams) ? "parameter" : "report";
+ final Map<String, String> reportParams = getReportParams(queryParams);
+ return switch (exportMode) {
+ case CSV -> exportCSV(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue);
+ case PDF -> exportPDF(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue);
+ case S3 -> exportS3(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue);
+ default -> exportJSON(reportName, queryParams, reportParams, isSelfServiceUserReport, parameterTypeValue,
+ exportMode == DatatableExportTargetParameter.PRETTY_JSON);
+ };
+ }
- // PDF format
- if (exportPdf) {
- final Map<String, String> reportParams = getReportParams(queryParams);
- final String pdfFileName = this.readExtraDataAndReportingService.retrieveReportPDF(reportName, parameterTypeValue, reportParams,
- isSelfServiceUserReport);
+ private Response exportJSON(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
+ boolean isSelfServiceUserReport, String parameterTypeValue, boolean prettyPrint) {
+ final GenericResultsetData result = this.readExtraDataAndReportingService.retrieveGenericResultset(reportName, parameterTypeValue,
+ reportParams, isSelfServiceUserReport);
- final File file = new File(pdfFileName);
+ String json;
+ final boolean genericResultSetIsPassed = ApiParameterHelper.genericResultSetPassed(queryParams);
+ final boolean genericResultSet = ApiParameterHelper.genericResultSet(queryParams);
+ if (genericResultSetIsPassed) {
+ if (genericResultSet) {
+ json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
+ } else {
+ json = this.genericDataService.generateJsonFromGenericResultsetData(result);
+ }
+ } else {
+ json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
+ }
- final ResponseBuilder response = Response.ok(file);
- response.header("Content-Disposition", "attachment; filename=\"" + pdfFileName + "\"");
- response.header("content-Type", "application/pdf");
+ return Response.ok().entity(json).type(MediaType.APPLICATION_JSON).build();
+ }
- return response.build();
- }
+ private Response exportS3(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
+ boolean isSelfServiceUserReport, String parameterTypeValue) {
+ throw new UnsupportedOperationException("S3 export not supported for datatables");
+ }
- // JSON format
- if (!exportCsv) {
- final Map<String, String> reportParams = getReportParams(queryParams);
-
- final GenericResultsetData result = this.readExtraDataAndReportingService.retrieveGenericResultset(reportName,
- parameterTypeValue, reportParams, isSelfServiceUserReport);
-
- String json;
- final boolean genericResultSetIsPassed = ApiParameterHelper.genericResultSetPassed(queryParams);
- final boolean genericResultSet = ApiParameterHelper.genericResultSet(queryParams);
- if (genericResultSetIsPassed) {
- if (genericResultSet) {
- json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
- } else {
- json = this.genericDataService.generateJsonFromGenericResultsetData(result);
- }
- } else {
- json = this.toApiJsonSerializer.serializePretty(prettyPrint, result);
- }
+ private Response exportPDF(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
+ boolean isSelfServiceUserReport, String parameterTypeValue) {
- return Response.ok().entity(json).type(MediaType.APPLICATION_JSON).build();
- }
+ final String pdfFileName = this.readExtraDataAndReportingService.retrieveReportPDF(reportName, parameterTypeValue, reportParams,
+ isSelfServiceUserReport);
- // CSV format
- final Map<String, String> reportParams = getReportParams(queryParams);
+ final File file = new File(pdfFileName);
+
+ final ResponseBuilder response = Response.ok(file);
+ response.header("Content-Disposition", "attachment; filename=\"" + pdfFileName + "\"");
+ response.header("content-Type", "application/pdf");
+
+ return response.build();
+ }
+
+ private Response exportCSV(String reportName, MultivaluedMap<String, String> queryParams, Map<String, String> reportParams,
+ boolean isSelfServiceUserReport, String parameterTypeValue) {
final StreamingOutput result = this.readExtraDataAndReportingService.retrieveReportCSV(reportName, parameterTypeValue, reportParams,
isSelfServiceUserReport);
return Response.ok().entity(result).type("text/csv")
.header("Content-Disposition", "attachment;filename=" + reportName.replaceAll(" ", "") + ".csv").build();
+
}
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java
index 6d3ec207e..a1958c7bd 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java
@@ -22,11 +22,12 @@ import com.lowagie.text.Document;
import com.lowagie.text.PageSize;
import com.lowagie.text.pdf.PdfPTable;
import com.lowagie.text.pdf.PdfWriter;
-import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
-import java.io.InputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -39,6 +40,8 @@ import java.util.Set;
import javax.ws.rs.core.StreamingOutput;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.lang3.StringUtils;
import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.JdbcSupport;
@@ -79,75 +82,27 @@ public class ReadReportingServiceImpl implements ReadReportingService {
final boolean isSelfServiceUserReport) {
return out -> {
try {
-
final GenericResultsetData result = retrieveGenericResultset(name, type, queryParams, isSelfServiceUserReport);
- final StringBuilder sb = generateCsvFileBuffer(result);
-
- final InputStream in = new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8));
-
- final byte[] outputByte = new byte[4096];
- Integer readLen = in.read(outputByte, 0, 4096);
-
- while (readLen != -1) {
- out.write(outputByte, 0, readLen);
- readLen = in.read(outputByte, 0, 4096);
- }
- // in.close();
- // out.flush();
- // out.close();
+ generateCsvFileBuffer(result, out);
} catch (final Exception e) {
throw new PlatformDataIntegrityException("error.msg.exception.error", e.getMessage(), e);
}
};
}
- private StringBuilder generateCsvFileBuffer(final GenericResultsetData result) {
- final StringBuilder writer = new StringBuilder();
-
- final List<ResultsetColumnHeaderData> columnHeaders = result.getColumnHeaders();
- log.debug("NO. of Columns: {}", columnHeaders.size());
- final Integer chSize = columnHeaders.size();
- for (int i = 0; i < chSize; i++) {
- writer.append('"' + columnHeaders.get(i).getColumnName() + '"');
- if (i < (chSize - 1)) {
- writer.append(",");
+ private void generateCsvFileBuffer(final GenericResultsetData result, OutputStream out) throws IOException {
+ try (CSVPrinter printer = new CSVPrinter(new OutputStreamWriter(out, StandardCharsets.UTF_8), CSVFormat.EXCEL)) {
+ final List<ResultsetColumnHeaderData> columnHeaders = result.getColumnHeaders();
+ final List<ResultsetRowData> data = result.getData();
+ final List<String> header = new ArrayList<>();
+ for (final ResultsetColumnHeaderData columnHeader : columnHeaders) {
+ header.add(columnHeader.getColumnName());
}
- }
- writer.append('\n');
-
- final List<ResultsetRowData> data = result.getData();
- List<Object> row;
- Integer rSize;
- // String currCol;
- String currColType;
- String currVal;
- final String doubleQuote = "\"";
- final String twoDoubleQuotes = doubleQuote + doubleQuote;
- log.debug("NO. of Rows: {}", data.size());
- for (ResultsetRowData element : data) {
- row = element.getRow();
- rSize = row.size();
- for (int j = 0; j < rSize; j++) {
- // currCol = columnHeaders.get(j).getColumnName();
- currColType = columnHeaders.get(j).getColumnType();
- currVal = (String) row.get(j);
- if (currVal != null) {
- if (currColType.equals("DECIMAL") || currColType.equals("DOUBLE") || currColType.equals("BIGINT")
- || currColType.equals("SMALLINT") || currColType.equals("INT")) {
- writer.append(currVal);
- } else {
- writer.append('"' + this.genericDataService.replace(currVal, doubleQuote, twoDoubleQuotes) + '"');
- }
-
- }
- if (j < (rSize - 1)) {
- writer.append(",");
- }
+ printer.printRecord(header);
+ for (final ResultsetRowData row : data) {
+ printer.printRecord(row.getRow());
}
- writer.append('\n');
}
-
- return writer;
}
@Override