You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fineract.apache.org by al...@apache.org on 2022/11/23 19:28:57 UTC
[fineract] branch develop updated: FINERACT-1794: Address improvement in file upload APIs
This is an automated email from the ASF dual-hosted git repository.
aleks pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/fineract.git
The following commit(s) were added to refs/heads/develop by this push:
new 5338263bd FINERACT-1794: Address improvement in file upload APIs
5338263bd is described below
commit 5338263bd8697cad62c21760fac28b69d93910b0
Author: Aleks <al...@apache.org>
AuthorDate: Tue Nov 22 02:03:44 2022 +0100
FINERACT-1794: Address improvement in file upload APIs
---
.../groovy/org.apache.fineract.dependencies.gradle | 32 +++++
fineract-provider/dependencies.gradle | 2 +
.../service/BulkImportWorkbookServiceImpl.java | 10 +-
.../jobs/executeemail/ExecuteEmailConfig.java | 6 +-
.../jobs/executeemail/ExecuteEmailTasklet.java | 5 +-
.../ExecuteReportMailingJobsConfig.java | 5 +-
.../ExecuteReportMailingJobsTasklet.java | 5 +-
.../core/config/ContentS3Config.java | 44 +++++++
.../core/config/FineractProperties.java | 30 +++++
.../service/ReadReportingServiceImpl.java | 5 +-
.../documentmanagement/api/ImagesApiResource.java | 23 +---
.../contentrepository/ContentPathSanitizer.java | 28 +++++
.../ContentRepositoryFactory.java | 30 ++---
.../FileSystemContentPathSanitizer.java | 138 +++++++++++++++++++++
.../FileSystemContentRepository.java | 65 ++++++----
.../contentrepository/S3ContentRepository.java | 28 ++---
.../src/main/resources/application.properties | 11 ++
.../src/test/resources/application-test.properties | 10 ++
.../integrationtests/client/ImageTest.java | 136 ++++++++++++++++++--
.../test/resources/image-gif-correct-extension.gif | Bin 0 -> 1018890 bytes
.../test/resources/image-gif-wrong-extension.png | Bin 0 -> 1018890 bytes
.../test/resources/image-text-wrong-content.jsp | 21 ++++
22 files changed, 532 insertions(+), 102 deletions(-)
diff --git a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle
index 04ed207cf..f91a1e7ad 100644
--- a/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle
+++ b/buildSrc/src/main/groovy/org.apache.fineract.dependencies.gradle
@@ -66,6 +66,38 @@ dependencyManagement {
dependency 'com.github.spullara.mustache.java:compiler:0.9.10'
dependency 'com.jayway.jsonpath:json-path:2.7.0'
dependency 'org.apache.tika:tika-core:2.6.0'
+ dependency ('org.apache.tika:tika-parser-microsoft-module:2.6.0') {
+ exclude 'org.bouncycastle:bcprov-jdk15on'
+ exclude 'org.bouncycastle:bcmail-jdk15on'
+ exclude 'commons-logging:commons-logging'
+ exclude 'org.apache.logging.log4j:log4j-api'
+ exclude 'org.slf4j:slf4j-api'
+ exclude 'commons-io:commons-io'
+ exclude 'commons-codec:commons-codec'
+ exclude 'org.apache.commons:commons-compress'
+ exclude 'org.apache.commons:commons-lang3'
+ exclude 'org.apache.poi:poi'
+ exclude 'org.apache.poi:poi-scratchpad'
+ exclude 'org.glassfish.jaxb:jaxb-runtime'
+ exclude 'org.apache.commons:commons-compress'
+ exclude 'xml-apis:xml-apis'
+ }
+ dependency ('org.apache.tika:tika-parser-miscoffice-module:2.6.0') {
+ exclude 'org.bouncycastle:bcprov-jdk15on'
+ exclude 'org.bouncycastle:bcmail-jdk15on'
+ exclude 'commons-logging:commons-logging'
+ exclude 'org.apache.logging.log4j:log4j-api'
+ exclude 'org.slf4j:slf4j-api'
+ exclude 'commons-io:commons-io'
+ exclude 'commons-codec:commons-codec'
+ exclude 'org.apache.commons:commons-compress'
+ exclude 'org.apache.commons:commons-lang3'
+ exclude 'org.apache.poi:poi'
+ exclude 'org.apache.poi:poi-scratchpad'
+ exclude 'org.glassfish.jaxb:jaxb-runtime'
+ exclude 'org.apache.commons:commons-compress'
+ exclude 'xml-apis:xml-apis'
+ }
dependency 'org.apache.httpcomponents:httpclient:4.5.13'
dependency 'jakarta.management.j2ee:jakarta.management.j2ee-api:1.1.4'
dependency 'jakarta.jms:jakarta.jms-api:2.0.3'
diff --git a/fineract-provider/dependencies.gradle b/fineract-provider/dependencies.gradle
index e8526710e..05c0e899f 100644
--- a/fineract-provider/dependencies.gradle
+++ b/fineract-provider/dependencies.gradle
@@ -63,6 +63,8 @@ dependencies {
'org.apache.poi:poi',
'org.apache.poi:poi-ooxml',
'org.apache.tika:tika-core',
+ 'org.apache.tika:tika-parser-microsoft-module',
+ 'org.apache.tika:tika-parser-miscoffice-module',
'org.liquibase:liquibase-core',
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookServiceImpl.java
index 393fccbf1..26af6968d 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookServiceImpl.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookServiceImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.fineract.infrastructure.bulkimport.service;
+import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -87,14 +88,13 @@ public class BulkImportWorkbookServiceImpl implements BulkImportWorkbookService
final String dateFormat) {
try {
if (entity != null && inputStream != null && fileDetail != null && locale != null && dateFormat != null) {
-
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
IOUtils.copy(inputStream, baos);
final byte[] bytes = baos.toByteArray();
InputStream clonedInputStream = new ByteArrayInputStream(bytes);
- InputStream clonedInputStreamWorkbook = new ByteArrayInputStream(bytes);
+ final BufferedInputStream bis = new BufferedInputStream(new ByteArrayInputStream(bytes));
final Tika tika = new Tika();
- final TikaInputStream tikaInputStream = TikaInputStream.get(clonedInputStream);
+ final TikaInputStream tikaInputStream = TikaInputStream.get(bis);
final String fileType = tika.detect(tikaInputStream);
if (!fileType.contains("msoffice") && !fileType.contains("application/vnd.ms-excel")) {
// We had a problem where we tried to upload the downloaded
@@ -104,7 +104,7 @@ public class BulkImportWorkbookServiceImpl implements BulkImportWorkbookService
"Uploaded file extension is not recognized.");
}
- Workbook workbook = new HSSFWorkbook(clonedInputStreamWorkbook);
+ Workbook workbook = new HSSFWorkbook(clonedInputStream);
GlobalEntityType entityType = null;
int primaryColumn = 0;
if (entity.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_PERSON.toString())) {
@@ -169,7 +169,7 @@ public class BulkImportWorkbookServiceImpl implements BulkImportWorkbookService
throw new GeneralPlatformDomainRuleException("error.msg.unable.to.find.resource", "Unable to find requested resource");
}
- return publishEvent(primaryColumn, fileDetail, clonedInputStreamWorkbook, entityType, workbook, locale, dateFormat);
+ return publishEvent(primaryColumn, fileDetail, bis, entityType, workbook, locale, dateFormat);
}
throw new GeneralPlatformDomainRuleException("error.msg.null", "One or more of the given parameters not found");
} catch (IOException e) {
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailConfig.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailConfig.java
index 7b0573eef..3ce76ffae 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailConfig.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailConfig.java
@@ -21,6 +21,7 @@ package org.apache.fineract.infrastructure.campaigns.jobs.executeemail;
import org.apache.fineract.infrastructure.campaigns.email.domain.EmailCampaignRepository;
import org.apache.fineract.infrastructure.campaigns.email.domain.EmailMessageRepository;
import org.apache.fineract.infrastructure.campaigns.email.service.EmailMessageJobEmailService;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
import org.apache.fineract.infrastructure.jobs.service.JobName;
import org.apache.fineract.infrastructure.reportmailingjob.validation.ReportMailingJobValidator;
@@ -57,6 +58,9 @@ public class ExecuteEmailConfig {
@Autowired
private ReportMailingJobValidator reportMailingJobValidator;
+ @Autowired
+ private FineractProperties fineractProperties;
+
@Bean
protected Step executeEmailStep() {
return steps.get(JobName.EXECUTE_EMAIL.name()).tasklet(executeEmailTasklet()).build();
@@ -70,6 +74,6 @@ public class ExecuteEmailConfig {
@Bean
public ExecuteEmailTasklet executeEmailTasklet() {
return new ExecuteEmailTasklet(emailMessageRepository, emailCampaignRepository, loanRepository, savingsAccountRepository,
- emailMessageJobEmailService, readReportingService, reportMailingJobValidator);
+ emailMessageJobEmailService, readReportingService, reportMailingJobValidator, fineractProperties);
}
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailTasklet.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailTasklet.java
index 1b5b4e92a..9cdac9bd7 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailTasklet.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executeemail/ExecuteEmailTasklet.java
@@ -39,10 +39,10 @@ import org.apache.fineract.infrastructure.campaigns.email.domain.EmailMessageRep
import org.apache.fineract.infrastructure.campaigns.email.domain.EmailMessageStatusType;
import org.apache.fineract.infrastructure.campaigns.email.domain.ScheduledEmailAttachmentFileFormat;
import org.apache.fineract.infrastructure.campaigns.email.service.EmailMessageJobEmailService;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.exception.PlatformDataIntegrityException;
import org.apache.fineract.infrastructure.dataqueries.domain.Report;
import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
-import org.apache.fineract.infrastructure.documentmanagement.contentrepository.FileSystemContentRepository;
import org.apache.fineract.infrastructure.reportmailingjob.helper.IPv4Helper;
import org.apache.fineract.infrastructure.reportmailingjob.validation.ReportMailingJobValidator;
import org.apache.fineract.portfolio.client.domain.Client;
@@ -66,6 +66,7 @@ public class ExecuteEmailTasklet implements Tasklet {
private final EmailMessageJobEmailService emailMessageJobEmailService;
private final ReadReportingService readReportingService;
private final ReportMailingJobValidator reportMailingJobValidator;
+ private final FineractProperties fineractProperties;
@Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
@@ -205,7 +206,7 @@ public class ExecuteEmailTasklet implements Tasklet {
try {
final ByteArrayOutputStream byteArrayOutputStream = readReportingService.generatePentahoReportAsOutputStream(reportName,
emailAttachmentFileFormat.getValue(), reportParams, null, emailCampaign.getApprovedBy(), errorLog);
- final String fileLocation = FileSystemContentRepository.FINERACT_BASE_DIR + File.separator + "";
+ final String fileLocation = fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator + "";
final String fileNameWithoutExtension = fileLocation + File.separator + reportName;
if (!new File(fileLocation).isDirectory()) {
new File(fileLocation).mkdirs();
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsConfig.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsConfig.java
index d2d734c52..9146a25b9 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsConfig.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsConfig.java
@@ -18,6 +18,7 @@
*/
package org.apache.fineract.infrastructure.campaigns.jobs.executereportmailingjobs;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
import org.apache.fineract.infrastructure.jobs.service.JobName;
import org.apache.fineract.infrastructure.report.provider.ReportingProcessServiceProvider;
@@ -54,6 +55,8 @@ public class ExecuteReportMailingJobsConfig {
private ReportMailingJobEmailService reportMailingJobEmailService;
@Autowired
private ReportMailingJobRunHistoryRepository reportMailingJobRunHistoryRepository;
+ @Autowired
+ private FineractProperties fineractProperties;
@Bean
protected Step executeReportMailingJobsStep() {
@@ -69,6 +72,6 @@ public class ExecuteReportMailingJobsConfig {
@Bean
public ExecuteReportMailingJobsTasklet executeReportMailingJobsTasklet() {
return new ExecuteReportMailingJobsTasklet(reportMailingJobRepository, reportMailingJobValidator, readReportingService,
- reportingProcessServiceProvider, reportMailingJobEmailService, reportMailingJobRunHistoryRepository);
+ reportingProcessServiceProvider, reportMailingJobEmailService, reportMailingJobRunHistoryRepository, fineractProperties);
}
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsTasklet.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsTasklet.java
index 584fb1033..68d85df01 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsTasklet.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/campaigns/jobs/executereportmailingjobs/ExecuteReportMailingJobsTasklet.java
@@ -34,10 +34,10 @@ import javax.ws.rs.core.Response;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.service.DateUtils;
import org.apache.fineract.infrastructure.dataqueries.domain.Report;
import org.apache.fineract.infrastructure.dataqueries.service.ReadReportingService;
-import org.apache.fineract.infrastructure.documentmanagement.contentrepository.FileSystemContentRepository;
import org.apache.fineract.infrastructure.report.provider.ReportingProcessServiceProvider;
import org.apache.fineract.infrastructure.report.service.ReportingProcessService;
import org.apache.fineract.infrastructure.reportmailingjob.data.ReportMailingJobEmailAttachmentFileFormat;
@@ -68,6 +68,7 @@ public class ExecuteReportMailingJobsTasklet implements Tasklet {
private final ReportingProcessServiceProvider reportingProcessServiceProvider;
private final ReportMailingJobEmailService reportMailingJobEmailService;
private final ReportMailingJobRunHistoryRepository reportMailingJobRunHistoryRepository;
+ private final FineractProperties fineractProperties;
private static final String DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
@@ -133,7 +134,7 @@ public class ExecuteReportMailingJobsTasklet implements Tasklet {
if (responseObject != null && responseObject.getClass().equals(ByteArrayOutputStream.class)) {
final ByteArrayOutputStream byteArrayOutputStream = (ByteArrayOutputStream) responseObject;
- final String fileLocation = FileSystemContentRepository.FINERACT_BASE_DIR + File.separator + "";
+ final String fileLocation = fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator + "";
final String fileNameWithoutExtension = fileLocation + File.separator + reportName;
if (!new File(fileLocation).isDirectory()) {
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/ContentS3Config.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/ContentS3Config.java
new file mode 100644
index 000000000..c4114d20c
--- /dev/null
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/ContentS3Config.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fineract.infrastructure.core.config;
+
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Slf4j
+@Configuration
+public class ContentS3Config {
+
+ @Bean
+ @ConditionalOnProperty("fineract.content.s3.enabled")
+ public AmazonS3 contentS3Client(FineractProperties fineractProperties) {
+ return AmazonS3ClientBuilder.standard()
+ .withCredentials(
+ new AWSStaticCredentialsProvider(new BasicAWSCredentials(fineractProperties.getContent().getS3().getAccessKey(),
+ fineractProperties.getContent().getS3().getSecretKey())))
+ .build();
+ }
+}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
index d0b70e6eb..c10e3ddd4 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/core/config/FineractProperties.java
@@ -45,6 +45,8 @@ public class FineractProperties {
private FineractEventsProperties events;
+ private FineractContentProperties content;
+
@Getter
@Setter
public static class FineractTenantProperties {
@@ -155,4 +157,32 @@ public class FineractProperties {
private String eventQueueName;
private String brokerUrl;
}
+
+ @Getter
+ @Setter
+ public static class FineractContentProperties {
+
+ private boolean regexWhitelistEnabled;
+ private List<String> regexWhitelist;
+ private boolean mimeWhitelistEnabled;
+ private List<String> mimeWhitelist;
+ private FineractContentFilesystemProperties filesystem;
+ private FineractContentS3Properties s3;
+ }
+
+ @Getter
+ @Setter
+ public static class FineractContentFilesystemProperties {
+
+ private String rootFolder;
+ }
+
+ @Getter
+ @Setter
+ public static class FineractContentS3Properties {
+
+ private String bucketName;
+ private String accessKey;
+ private String secretKey;
+ }
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java
index 1acab7c17..6d3ec207e 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/ReadReportingServiceImpl.java
@@ -40,6 +40,7 @@ import javax.ws.rs.core.StreamingOutput;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.JdbcSupport;
import org.apache.fineract.infrastructure.core.exception.PlatformDataIntegrityException;
import org.apache.fineract.infrastructure.core.service.database.DatabaseSpecificSQLGenerator;
@@ -50,7 +51,6 @@ import org.apache.fineract.infrastructure.dataqueries.data.ReportParameterJoinDa
import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData;
import org.apache.fineract.infrastructure.dataqueries.data.ResultsetRowData;
import org.apache.fineract.infrastructure.dataqueries.exception.ReportNotFoundException;
-import org.apache.fineract.infrastructure.documentmanagement.contentrepository.FileSystemContentRepository;
import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext;
import org.apache.fineract.infrastructure.security.service.SqlInjectionPreventerService;
import org.apache.fineract.infrastructure.security.utils.LogParameterEscapeUtil;
@@ -72,6 +72,7 @@ public class ReadReportingServiceImpl implements ReadReportingService {
private final GenericDataService genericDataService;
private final SqlInjectionPreventerService sqlInjectionPreventerService;
private final DatabaseSpecificSQLGenerator sqlGenerator;
+ private final FineractProperties fineractProperties;
@Override
public StreamingOutput retrieveReportCSV(final String name, final String type, final Map<String, String> queryParams,
@@ -240,7 +241,7 @@ public class ReadReportingServiceImpl implements ReadReportingService {
public String retrieveReportPDF(final String reportName, final String type, final Map<String, String> queryParams,
final boolean isSelfServiceUserReport) {
- final String fileLocation = FileSystemContentRepository.FINERACT_BASE_DIR + File.separator + "";
+ final String fileLocation = fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator + "";
if (!new File(fileLocation).isDirectory()) {
new File(fileLocation).mkdirs();
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/api/ImagesApiResource.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/api/ImagesApiResource.java
index fcf1208cc..bf010e84c 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/api/ImagesApiResource.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/api/ImagesApiResource.java
@@ -36,6 +36,8 @@ import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.fineract.infrastructure.core.data.CommandProcessingResult;
import org.apache.fineract.infrastructure.core.data.UploadRequest;
@@ -54,19 +56,16 @@ import org.apache.fineract.portfolio.client.data.ClientData;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataParam;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+@Slf4j
+@RequiredArgsConstructor
@Component
@Scope("singleton")
@Path("{entity}/{entityId}/images")
public class ImagesApiResource {
- private static final Logger LOG = LoggerFactory.getLogger(ImagesApiResource.class);
-
private final PlatformSecurityContext context;
private final ImageReadPlatformService imageReadPlatformService;
private final ImageWritePlatformService imageWritePlatformService;
@@ -74,18 +73,6 @@ public class ImagesApiResource {
private final FileUploadValidator fileUploadValidator;
private final ImageResizer imageResizer;
- @Autowired
- public ImagesApiResource(final PlatformSecurityContext context, final ImageReadPlatformService readPlatformService,
- final ImageWritePlatformService imageWritePlatformService, final DefaultToApiJsonSerializer<ClientData> toApiJsonSerializer,
- final FileUploadValidator fileUploadValidator, final ImageResizer imageResizer) {
- this.context = context;
- this.imageReadPlatformService = readPlatformService;
- this.imageWritePlatformService = imageWritePlatformService;
- this.toApiJsonSerializer = toApiJsonSerializer;
- this.fileUploadValidator = fileUploadValidator;
- this.imageResizer = imageResizer;
- }
-
/**
* Upload images through multi-part form upload
*/
@@ -172,7 +159,7 @@ public class ImagesApiResource {
final String clientImageAsBase64Text = imageDataURISuffix + Base64.getMimeEncoder().encodeToString(resizedImageBytes);
return Response.ok(clientImageAsBase64Text, MediaType.TEXT_PLAIN_TYPE).build();
} else {
- LOG.error("resizedImageBytes is null for entityName={}, entityId={}, maxWidth={}, maxHeight={}", entityName, entityId,
+ log.error("resizedImageBytes is null for entityName={}, entityId={}, maxWidth={}, maxHeight={}", entityName, entityId,
maxWidth, maxHeight);
return Response.serverError().build();
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentPathSanitizer.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentPathSanitizer.java
new file mode 100644
index 000000000..415532fcd
--- /dev/null
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentPathSanitizer.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.documentmanagement.contentrepository;
+
+import java.io.BufferedInputStream;
+
+public interface ContentPathSanitizer {
+
+ String sanitize(String path);
+
+ String sanitize(String path, BufferedInputStream is);
+}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentRepositoryFactory.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentRepositoryFactory.java
index bdfcb5c69..5c8fa8f90 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentRepositoryFactory.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/ContentRepositoryFactory.java
@@ -18,40 +18,28 @@
*/
package org.apache.fineract.infrastructure.documentmanagement.contentrepository;
+import java.util.List;
import lombok.RequiredArgsConstructor;
-import org.apache.fineract.infrastructure.configuration.data.S3CredentialsData;
import org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
-import org.apache.fineract.infrastructure.configuration.service.ExternalServicesPropertiesReadPlatformService;
import org.apache.fineract.infrastructure.documentmanagement.domain.StorageType;
-import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component;
@Component
@RequiredArgsConstructor
public class ContentRepositoryFactory {
- private final ApplicationContext applicationContext;
- private final ExternalServicesPropertiesReadPlatformService externalServicesReadPlatformService;
+ // TODO: all configuration should be really moved to application.properties
+ private final ConfigurationDomainService configurationService;
+ private final List<ContentRepository> contentRepositories;
public ContentRepository getRepository() {
- final ConfigurationDomainService configurationDomainServiceJpa = this.applicationContext.getBean("configurationDomainServiceJpa",
- ConfigurationDomainService.class);
- if (configurationDomainServiceJpa.isAmazonS3Enabled()) {
- return createS3DocumentStore();
+ if (configurationService.isAmazonS3Enabled()) {
+ return getRepository(StorageType.S3);
}
- return new FileSystemContentRepository();
+ return getRepository(StorageType.FILE_SYSTEM);
}
- public ContentRepository getRepository(final StorageType documentStoreType) {
- if (documentStoreType == StorageType.FILE_SYSTEM) {
- return new FileSystemContentRepository();
- }
- return createS3DocumentStore();
- }
-
- private ContentRepository createS3DocumentStore() {
- final S3CredentialsData s3CredentialsData = this.externalServicesReadPlatformService.getS3Credentials();
- return new S3ContentRepository(s3CredentialsData.getBucketName(), s3CredentialsData.getSecretKey(),
- s3CredentialsData.getAccessKey());
+ public ContentRepository getRepository(StorageType storageType) {
+ return contentRepositories.stream().filter(cr -> cr.getStorageType().equals(storageType)).findFirst().orElseThrow();
}
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentPathSanitizer.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentPathSanitizer.java
new file mode 100644
index 000000000..b25fc97c4
--- /dev/null
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentPathSanitizer.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.fineract.infrastructure.documentmanagement.contentrepository;
+
+import java.io.BufferedInputStream;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.regex.Pattern;
+import javax.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
+import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
+import org.apache.fineract.infrastructure.documentmanagement.exception.ContentManagementException;
+import org.apache.tika.Tika;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.sax.BodyContentHandler;
+import org.springframework.stereotype.Component;
+
+@Slf4j
+@RequiredArgsConstructor
+@Component
+public class FileSystemContentPathSanitizer implements ContentPathSanitizer {
+
+ private final FineractProperties fineractProperties;
+
+ private List<Pattern> regexWhitelist;
+
+ private static Pattern OVERWRITE_SIBLING_IMAGE = Pattern.compile(".*\\.\\./+[0-9]+/+.*");
+
+ @PostConstruct
+ public void init() {
+ regexWhitelist = fineractProperties.getContent().getRegexWhitelist().stream().map(Pattern::compile).toList();
+ }
+
+ @Override
+ public String sanitize(String path) {
+ return sanitize(path, null);
+ }
+
+ @Override
+ public String sanitize(String path, BufferedInputStream is) {
+ try {
+ if (OVERWRITE_SIBLING_IMAGE.matcher(path).matches()) {
+ throw new RuntimeException(String.format("Trying to overwrite another resource's image: %s", path));
+ }
+
+ String sanitizedPath = Path.of(path).normalize().toString();
+
+ String fileName = FilenameUtils.getName(sanitizedPath).toLowerCase();
+
+ if (log.isDebugEnabled()) {
+ log.debug("Path: {} -> {} ({})", path, sanitizedPath, fileName);
+ }
+
+ if (fineractProperties.getContent().isRegexWhitelistEnabled()) {
+ boolean matches = regexWhitelist.stream().anyMatch(p -> p.matcher(fileName).matches());
+
+ if (!matches) {
+ throw new RuntimeException(String.format("File name not allowed: %s", fileName));
+ }
+ }
+
+ if (is != null && fineractProperties.getContent().isMimeWhitelistEnabled()) {
+ Tika tika = new Tika();
+ String extensionMimeType = tika.detect(fileName);
+
+ if (StringUtils.isEmpty(extensionMimeType)) {
+ throw new RuntimeException(String.format("Could not detect mime type for filename %s!", fileName));
+ }
+
+ if (!fineractProperties.getContent().getMimeWhitelist().contains(extensionMimeType)) {
+ throw new RuntimeException(
+ String.format("Detected mime type %s for filename %s not allowed!", extensionMimeType, fileName));
+ }
+
+ String contentMimeType = detectContentMimeType(is);
+
+ if (StringUtils.isEmpty(contentMimeType)) {
+ throw new RuntimeException(String.format("Could not detect content mime type for %s!", fileName));
+ }
+
+ if (!fineractProperties.getContent().getMimeWhitelist().contains(contentMimeType)) {
+ throw new RuntimeException(
+ String.format("Detected content mime type %s for %s not allowed!", contentMimeType, fileName));
+ }
+
+ if (!contentMimeType.equalsIgnoreCase(extensionMimeType)) {
+ throw new RuntimeException(String.format("Detected filename (%s) and content (%s) mime type do not match!",
+ extensionMimeType, contentMimeType));
+ }
+ }
+
+ Path target = Path.of(sanitizedPath);
+ Path rootFolder = Path.of(fineractProperties.getContent().getFilesystem().getRootFolder(),
+ ThreadLocalContextUtil.getTenant().getName().replaceAll(" ", "").trim());
+
+ if (!target.startsWith(rootFolder)) {
+ throw new RuntimeException(String.format("Path traversal attempt: %s (%s)", target, rootFolder));
+ }
+
+ return sanitizedPath;
+ } catch (Exception e) {
+ throw new ContentManagementException(path, e.getMessage(), e);
+ }
+ }
+
+ private String detectContentMimeType(BufferedInputStream bis) throws Exception {
+ TikaInputStream tis = TikaInputStream.get(bis);
+ AutoDetectParser parser = new AutoDetectParser();
+ // NOTE: turn off write limit with "-1"
+ BodyContentHandler handler = new BodyContentHandler(-1);
+ Metadata metadata = new Metadata();
+ parser.parse(tis, handler, metadata);
+
+ return metadata.get("Content-Type");
+ }
+}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentRepository.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentRepository.java
index 19a83e1fe..1329f87ea 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentRepository.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/FileSystemContentRepository.java
@@ -19,12 +19,16 @@
package org.apache.fineract.infrastructure.documentmanagement.contentrepository;
import com.google.common.io.Files;
+import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Base64;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.Base64EncodedImage;
import org.apache.fineract.infrastructure.core.service.ThreadLocalContextUtil;
import org.apache.fineract.infrastructure.documentmanagement.command.DocumentCommand;
@@ -33,14 +37,17 @@ import org.apache.fineract.infrastructure.documentmanagement.data.FileData;
import org.apache.fineract.infrastructure.documentmanagement.data.ImageData;
import org.apache.fineract.infrastructure.documentmanagement.domain.StorageType;
import org.apache.fineract.infrastructure.documentmanagement.exception.ContentManagementException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+@Slf4j
+@RequiredArgsConstructor
+@Component
+@ConditionalOnProperty("fineract.content.filesystem.enabled")
public class FileSystemContentRepository implements ContentRepository {
- private static final Logger LOG = LoggerFactory.getLogger(FileSystemContentRepository.class);
-
- public static final String FINERACT_BASE_DIR = System.getProperty("user.home") + File.separator + ".fineract";
+ private final FileSystemContentPathSanitizer pathSanitizer;
+ private final FineractProperties fineractProperties;
@Override
public String saveFile(final InputStream uploadedInputStream, final DocumentCommand documentCommand) {
@@ -50,16 +57,14 @@ public class FileSystemContentRepository implements ContentRepository {
final String fileLocation = generateFileParentDirectory(documentCommand.getParentEntityType(), documentCommand.getParentEntityId())
+ File.separator + fileName;
- writeFileToFileSystem(fileName, uploadedInputStream, fileLocation);
- return fileLocation;
+ return writeFileToFileSystem(fileName, uploadedInputStream, fileLocation);
}
@Override
public String saveImage(final InputStream uploadedInputStream, final Long resourceId, final String imageName, final Long fileSize) {
ContentRepositoryUtils.validateFileSizeWithinPermissibleRange(fileSize, imageName);
final String fileLocation = generateClientImageParentDirectory(resourceId) + File.separator + imageName;
- writeFileToFileSystem(imageName, uploadedInputStream, fileLocation);
- return fileLocation;
+ return writeFileToFileSystem(imageName, uploadedInputStream, fileLocation);
}
@Override
@@ -69,10 +74,9 @@ public class FileSystemContentRepository implements ContentRepository {
String base64EncodedImageString = base64EncodedImage.getBase64EncodedString();
try {
final InputStream toUploadInputStream = new ByteArrayInputStream(Base64.getMimeDecoder().decode(base64EncodedImageString));
- writeFileToFileSystem(imageName, toUploadInputStream, fileLocation);
- return fileLocation;
+ return writeFileToFileSystem(imageName, toUploadInputStream, fileLocation);
} catch (IllegalArgumentException iae) {
- LOG.error("IllegalArgumentException due to invalid Base64 encoding: {}", base64EncodedImageString, iae);
+ log.error("IllegalArgumentException due to invalid Base64 encoding: {}", base64EncodedImageString, iae);
throw iae;
}
}
@@ -88,23 +92,29 @@ public class FileSystemContentRepository implements ContentRepository {
}
private void deleteFileInternal(final String documentPath) {
- final File fileToBeDeleted = new File(documentPath);
+ String sanitizedPath = pathSanitizer.sanitize(documentPath);
+
+ final File fileToBeDeleted = new File(sanitizedPath);
final boolean fileDeleted = fileToBeDeleted.delete();
if (!fileDeleted) {
// no need to throw an Error, what's a caller going to do about it, so simply log a warning
- LOG.warn("Unable to delete file {}", documentPath);
+ log.warn("Unable to delete file {}", documentPath);
}
}
@Override
public FileData fetchFile(final DocumentData documentData) {
- final File file = new File(documentData.fileLocation());
+ String sanitizedPath = pathSanitizer.sanitize(documentData.fileLocation());
+
+ final File file = new File(sanitizedPath);
return new FileData(Files.asByteSource(file), documentData.fileName(), documentData.contentType());
}
@Override
public FileData fetchImage(final ImageData imageData) {
- final File file = new File(imageData.location());
+ String sanitizedPath = pathSanitizer.sanitize(imageData.location());
+
+ final File file = new File(sanitizedPath);
return new FileData(Files.asByteSource(file), imageData.getEntityDisplayName(), imageData.contentType().getValue());
}
@@ -117,16 +127,17 @@ public class FileSystemContentRepository implements ContentRepository {
* Generate the directory path for storing the new document
*/
private String generateFileParentDirectory(final String entityType, final Long entityId) {
- return FileSystemContentRepository.FINERACT_BASE_DIR + File.separator
+ return fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator
+ ThreadLocalContextUtil.getTenant().getName().replaceAll(" ", "").trim() + File.separator + "documents" + File.separator
+ entityType + File.separator + entityId + File.separator + ContentRepositoryUtils.generateRandomString();
}
/**
- * Generate directory path for storing new Image
+ * Generate ContentRepositoryUtilsfineractProperties.getContentgetWhitelist()getBlacklist() path for storing new
+ * Image
*/
private String generateClientImageParentDirectory(final Long resourceId) {
- return FileSystemContentRepository.FINERACT_BASE_DIR + File.separator
+ return fineractProperties.getContent().getFilesystem().getRootFolder() + File.separator
+ ThreadLocalContextUtil.getTenant().getName().replaceAll(" ", "").trim() + File.separator + "images" + File.separator
+ "clients" + File.separator + resourceId;
}
@@ -135,16 +146,18 @@ public class FileSystemContentRepository implements ContentRepository {
* Recursively create the directory if it does not exist.
*/
private void makeDirectories(final String uploadDocumentLocation) throws IOException {
- Files.createParentDirs(new File(uploadDocumentLocation));
+ String sanitizedPath = pathSanitizer.sanitize(uploadDocumentLocation);
+ Files.createParentDirs(new File(sanitizedPath));
}
- private void writeFileToFileSystem(final String fileName, final InputStream uploadedInputStream, final String fileLocation) {
- try {
- makeDirectories(fileLocation);
- FileUtils.copyInputStreamToFile(uploadedInputStream, new File(fileLocation)); // NOSONAR
+ private String writeFileToFileSystem(final String fileName, final InputStream uploadedInputStream, final String fileLocation) {
+ try (BufferedInputStream bis = new BufferedInputStream(uploadedInputStream)) {
+ String sanitizedPath = pathSanitizer.sanitize(fileLocation, bis);
+ makeDirectories(sanitizedPath);
+ FileUtils.copyInputStreamToFile(bis, new File(sanitizedPath)); // NOSONAR
+ return sanitizedPath;
} catch (final IOException ioException) {
- LOG.warn("writeFileToFileSystem() IOException (logged because cause is not propagated in ContentManagementException)",
- ioException);
+ log.warn("Failed to write file!", ioException);
throw new ContentManagementException(fileName, ioException.getMessage(), ioException);
}
}
diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/S3ContentRepository.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/S3ContentRepository.java
index 38bfdc9b7..0a91f4ad9 100644
--- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/S3ContentRepository.java
+++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/documentmanagement/contentrepository/S3ContentRepository.java
@@ -20,10 +20,7 @@ package org.apache.fineract.infrastructure.documentmanagement.contentrepository;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.DeleteObjectRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ObjectMetadata;
@@ -35,7 +32,9 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Base64;
+import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
+import org.apache.fineract.infrastructure.core.config.FineractProperties;
import org.apache.fineract.infrastructure.core.domain.Base64EncodedImage;
import org.apache.fineract.infrastructure.documentmanagement.command.DocumentCommand;
import org.apache.fineract.infrastructure.documentmanagement.data.DocumentData;
@@ -45,18 +44,17 @@ import org.apache.fineract.infrastructure.documentmanagement.domain.StorageType;
import org.apache.fineract.infrastructure.documentmanagement.exception.ContentManagementException;
import org.apache.fineract.infrastructure.documentmanagement.exception.DocumentNotFoundException;
import org.apache.fineract.infrastructure.security.utils.LogParameterEscapeUtil;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
@Slf4j
+@RequiredArgsConstructor
+@Component
+@ConditionalOnProperty("fineract.content.s3.enabled")
public class S3ContentRepository implements ContentRepository {
- private final String s3BucketName;
private final AmazonS3 s3Client;
-
- public S3ContentRepository(final String bucketName, final String secretKey, final String accessKey) {
- this.s3BucketName = bucketName;
- this.s3Client = AmazonS3ClientBuilder.standard()
- .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey))).build();
- }
+ private final FineractProperties fineractProperties;
@Override
public String saveFile(final InputStream toUpload, final DocumentCommand documentCommand) {
@@ -142,7 +140,7 @@ public class S3ContentRepository implements ContentRepository {
private void deleteObject(final String location) {
try {
- this.s3Client.deleteObject(new DeleteObjectRequest(this.s3BucketName, location));
+ this.s3Client.deleteObject(new DeleteObjectRequest(fineractProperties.getContent().getS3().getBucketName(), location));
} catch (final AmazonServiceException ase) {
throw new ContentManagementException(location, "message=" + ase.getMessage() + ", Error Type=" + ase.getErrorType(), ase);
} catch (final AmazonClientException ace) {
@@ -156,7 +154,8 @@ public class S3ContentRepository implements ContentRepository {
if (log.isDebugEnabled()) {
log.debug("Uploading a new object to S3 {}", LogParameterEscapeUtil.escapeLogParameter(s3UploadLocation));
}
- this.s3Client.putObject(new PutObjectRequest(this.s3BucketName, s3UploadLocation, inputStream, new ObjectMetadata()));
+ this.s3Client.putObject(new PutObjectRequest(fineractProperties.getContent().getS3().getBucketName(), s3UploadLocation,
+ inputStream, new ObjectMetadata()));
} catch (AmazonClientException ase) {
throw new ContentManagementException(filename, ase.getMessage(), ase);
}
@@ -164,8 +163,9 @@ public class S3ContentRepository implements ContentRepository {
private S3Object getObject(String key) {
try {
- log.debug("Downloading an object from Amazon S3 Bucket: {}, location: {}", this.s3BucketName, key);
- return this.s3Client.getObject(new GetObjectRequest(this.s3BucketName, key));
+ log.debug("Downloading an object from Amazon S3 Bucket: {}, location: {}",
+ fineractProperties.getContent().getS3().getBucketName(), key);
+ return this.s3Client.getObject(new GetObjectRequest(fineractProperties.getContent().getS3().getBucketName(), key));
} catch (AmazonClientException ase) {
throw new ContentManagementException(key, ase.getMessage(), ase);
}
diff --git a/fineract-provider/src/main/resources/application.properties b/fineract-provider/src/main/resources/application.properties
index bd1a93a11..db24d4f67 100644
--- a/fineract-provider/src/main/resources/application.properties
+++ b/fineract-provider/src/main/resources/application.properties
@@ -70,6 +70,17 @@ fineract.loan.transactionprocessor.principal-interest-penalties-fees.enabled=${F
fineract.loan.transactionprocessor.rbi-india.enabled=${FINERACT_LOAN_TRANSACTIONPROCESSOR_RBI_INDIA_ENABLED:true}
fineract.loan.transactionprocessor.error-not-found-fail=${FINERACT_LOAN_TRANSACTIONPROCESSOR_ERROR_NOT_FOUND_FAIL:true}
+fineract.content.regex-whitelist-enabled=${FINERACT_CONTENT_REGEX_WHITELIST_ENABLED:true}
+fineract.content.regex-whitelist=${FINERACT_CONTENT_REGEX_WHITELIST:.*\\.pdf$,.*\\.doc,.*\\.docx,.*\\.xls,.*\\.xlsx,.*\\.jpg,.*\\.jpeg,.*\\.png}
+fineract.content.mime-whitelist-enabled=${FINERACT_CONTENT_MIME_WHITELIST_ENABLED:true}
+fineract.content.mime-whitelist=${FINERACT_CONTENT_MIME_WHITELIST:application/pdf,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,image/jpeg,image/png}
+fineract.content.filesystem.enabled=${FINERACT_CONTENT_FILESYSTEM_ENABLED:true}
+fineract.content.filesystem.rootFolder=${FINERACT_CONTENT_FILESYSTEM_ROOT_FOLDER:${user.home}/.fineract}
+fineract.content.s3.enabled=${FINERACT_CONTENT_S3_ENABLED:false}
+fineract.content.s3.bucketName=${FINERACT_CONTENT_S3_BUCKET_NAME:}
+fineract.content.s3.accessKey=${FINERACT_CONTENT_S3_ACCESS_KEY:}
+fineract.content.s3.secretKey=${FINERACT_CONTENT_S3_SECRET_KEY:}
+
# Logging pattern for the console
logging.pattern.console=${CONSOLE_LOG_PATTERN:%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(%replace([%X{correlationId}]){'\\[\\]', ''}) %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:%wEx}}
diff --git a/fineract-provider/src/test/resources/application-test.properties b/fineract-provider/src/test/resources/application-test.properties
index d19755693..492a2aa65 100644
--- a/fineract-provider/src/test/resources/application-test.properties
+++ b/fineract-provider/src/test/resources/application-test.properties
@@ -60,6 +60,16 @@ fineract.loan.transactionprocessor.principal-interest-penalties-fees.enabled=tru
fineract.loan.transactionprocessor.rbi-india.enabled=true
fineract.loan.transactionprocessor.error-not-found-fail=true
+fineract.content.regex-whitelist-enabled=true
+fineract.content.regex-whitelist=.*\\.pdf$,.*\\.doc,.*\\.docx,.*\\.xls,.*\\.xlsx,.*\\.jpg,.*\\.jpeg,.*\\.png
+fineract.content.mime-whitelist-enabled=true
+fineract.content.mime-whitelist=application/pdf,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,image/jpeg,image/png
+fineract.content.filesystem.enabled=true
+fineract.content.filesystem.rootFolder=${user.home}/.fineract
+fineract.content.s3.enabled=false
+fineract.content.s3.bucketName=
+fineract.content.s3.accessKey=
+fineract.content.s3.secretKey=
management.health.jms.enabled=false
diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ImageTest.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ImageTest.java
index 8bf466bbb..0bad4fb36 100644
--- a/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ImageTest.java
+++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/client/ImageTest.java
@@ -18,10 +18,15 @@
*/
package org.apache.fineract.integrationtests.client;
-import java.io.File;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
import java.io.IOException;
+import lombok.extern.slf4j.Slf4j;
import okhttp3.MediaType;
+import okhttp3.MultipartBody;
+import okhttp3.RequestBody;
import okhttp3.ResponseBody;
+import org.apache.commons.io.IOUtils;
import org.apache.fineract.client.services.ImagesApi;
import org.apache.fineract.client.util.Parts;
import org.junit.jupiter.api.Order;
@@ -36,9 +41,10 @@ import retrofit2.http.Headers;
*
* @author Michael Vorburger.ch
*/
+@Slf4j
public class ImageTest extends IntegrationTest {
- final File testImage = new File(getClass().getResource("/michael.vorburger-crepes.jpg").getFile());
+ final MultipartBody.Part testPart = createPart("michael.vorburger-crepes.jpg", "michael.vorburger-crepes.jpg", "image/jpeg");
Long clientId = new ClientTest().getClientId();
Long staffId = new StaffTest().getStaffId();
@@ -46,8 +52,8 @@ public class ImageTest extends IntegrationTest {
@Test
@Order(1)
void create() {
- ok(fineract().images.create("staff", staffId, Parts.fromFile(testImage)));
- ok(fineract().images.create("clients", clientId, Parts.fromFile(testImage)));
+ ok(fineract().images.create("staff", staffId, testPart));
+ ok(fineract().images.create("clients", clientId, testPart));
}
@Test
@@ -82,8 +88,8 @@ public class ImageTest extends IntegrationTest {
Response<ResponseBody> r = okR(fineract().images.get("staff", staffId, 3505, 1972, "inline_octet"));
try (ResponseBody body = r.body()) {
assertThat(body.contentType()).isEqualTo(MediaType.get("image/jpeg"));
- assertThat(body.bytes().length).isEqualTo(testImage.length());
- assertThat(body.contentLength()).isEqualTo(testImage.length());
+ assertThat(body.bytes().length).isEqualTo(testPart.body().contentLength());
+ assertThat(body.contentLength()).isEqualTo(testPart.body().contentLength());
}
var staff = ok(fineract().staff.retrieveOne8(staffId));
@@ -96,8 +102,8 @@ public class ImageTest extends IntegrationTest {
void getOctetOutput() throws IOException {
ResponseBody r = ok(fineract().images.get("staff", staffId, 3505, 1972, "octet"));
assertThat(r.contentType()).isEqualTo(MediaType.get("image/jpeg"));
- assertThat(r.bytes().length).isEqualTo(testImage.length());
- assertThat(r.contentLength()).isEqualTo(testImage.length());
+ assertThat(r.bytes().length).isEqualTo(testPart.body().contentLength());
+ assertThat(r.contentLength()).isEqualTo(testPart.body().contentLength());
}
@Test
@@ -121,13 +127,13 @@ public class ImageTest extends IntegrationTest {
void getBytes() throws IOException {
ResponseBody r = ok(fineract().createService(ImagesApiWithHeadersForTest.class).getBytes("staff", staffId, 3505, 1972, null));
assertThat(r.contentType()).isEqualTo(MediaType.get("image/jpeg"));
- assertThat(r.bytes().length).isEqualTo(testImage.length());
+ assertThat(r.bytes().length).isEqualTo(testPart.body().contentLength());
}
@Test
@Order(50)
void update() {
- ok(fineract().images.update("staff", staffId, Parts.fromFile(testImage)));
+ ok(fineract().images.update("staff", staffId, testPart));
}
@Test
@@ -137,6 +143,116 @@ public class ImageTest extends IntegrationTest {
ok(fineract().images.delete("clients", clientId));
}
+ @Test
+ @Order(100)
+ void pathTraversalJsp() {
+ final MultipartBody.Part part = createPart("image-text-wrong-content.jsp",
+ "../../../../../../../../../../tmp/image-text-wrong-content.jsp", "image/gif");
+
+ assertThat(part).isNotNull();
+
+ Exception exception = assertThrows(Exception.class, () -> {
+ ok(fineract().images.create("clients", clientId, part));
+ });
+
+ assertThat(exception).isNotNull();
+
+ log.warn("Should not be able to upload a file that doesn't match the indicated content type: {}", exception.getMessage());
+ }
+
+ @Test
+ @Order(101)
+ void gifWithPngExtension() {
+ final MultipartBody.Part part = createPart("image-gif-wrong-extension.png", "image-gif-wrong-extension.png", "image/png");
+
+ assertThat(part).isNotNull();
+
+ Exception exception = assertThrows(Exception.class, () -> {
+ ok(fineract().images.create("clients", clientId, part));
+ });
+
+ assertThat(exception).isNotNull();
+
+ log.warn("Should not be able to upload a gif by just renaming the file extension: {}", exception.getMessage());
+ }
+
+ @Test
+ @Order(102)
+ void gifImage() {
+ final MultipartBody.Part part = createPart("image-gif-correct-extension.gif", "image-gif-correct-extension.gif", "image/png");
+
+ assertThat(part).isNotNull();
+
+ Exception exception = assertThrows(Exception.class, () -> {
+ ok(fineract().images.create("clients", clientId, part));
+ });
+
+ assertThat(exception).isNotNull();
+
+ log.warn("Should not be able to upload a gif it is not whitelisted: {}", exception.getMessage());
+ }
+
+ @Test
+ @Order(103)
+ void pathTraversalJpg() {
+ final MultipartBody.Part part = createPart("michael.vorburger-crepes.jpg",
+ "../../../../../../../../../../tmp/michael.vorburger-crepes.jpg", "image/jpeg");
+
+ assertThat(part).isNotNull();
+
+ Exception exception = assertThrows(Exception.class, () -> {
+ ok(fineract().images.create("clients", clientId, part));
+ });
+
+ assertThat(exception).isNotNull();
+
+ log.warn("Should not be able to upload a file with a forbidden name pattern: {}", exception.getMessage());
+ }
+
+ @Test
+ @Order(104)
+ void pathTraversalWithAbsolutePathJpg() {
+ final MultipartBody.Part part = createPart("michael.vorburger-crepes.jpg", "../17/michael.vorburger-crepes.jpg", "image/jpeg");
+
+ assertThat(part).isNotNull();
+
+ Exception exception = assertThrows(Exception.class, () -> {
+ ok(fineract().images.create("clients", clientId, part));
+ });
+
+ assertThat(exception).isNotNull();
+
+ log.warn("Should not be able to upload a file with a forbidden name pattern: {}", exception.getMessage());
+ }
+
+ @Test
+ @Order(105)
+ void pathTraversalWithAbsolutePathJpg2() {
+ final MultipartBody.Part part = createPart("michael.vorburger-crepes.jpg", "..//17//michael.vorburger-crepes.jpg", "image/jpeg");
+
+ assertThat(part).isNotNull();
+
+ Exception exception = assertThrows(Exception.class, () -> {
+ ok(fineract().images.create("clients", clientId, part));
+ });
+
+ assertThat(exception).isNotNull();
+
+ log.warn("Should not be able to upload a file with a forbidden name pattern: {}", exception.getMessage());
+ }
+
+ private MultipartBody.Part createPart(String fileResource, String fileName, String mediaType) {
+ try {
+ byte[] data = IOUtils.toByteArray(ImageTest.class.getClassLoader().getResourceAsStream(fileResource));
+ RequestBody rb = RequestBody.create(data, MediaType.get(mediaType));
+ return MultipartBody.Part.createFormData("file", fileName, rb);
+ } catch (Exception e) {
+ log.error("Error creating file part.", e);
+ }
+
+ return null;
+ }
+
interface ImagesApiWithHeadersForTest extends ImagesApi {
@Headers("Accept: text/plain")
diff --git a/integration-tests/src/test/resources/image-gif-correct-extension.gif b/integration-tests/src/test/resources/image-gif-correct-extension.gif
new file mode 100644
index 000000000..89b951662
Binary files /dev/null and b/integration-tests/src/test/resources/image-gif-correct-extension.gif differ
diff --git a/integration-tests/src/test/resources/image-gif-wrong-extension.png b/integration-tests/src/test/resources/image-gif-wrong-extension.png
new file mode 100644
index 000000000..89b951662
Binary files /dev/null and b/integration-tests/src/test/resources/image-gif-wrong-extension.png differ
diff --git a/integration-tests/src/test/resources/image-text-wrong-content.jsp b/integration-tests/src/test/resources/image-text-wrong-content.jsp
new file mode 100644
index 000000000..6e660673c
--- /dev/null
+++ b/integration-tests/src/test/resources/image-text-wrong-content.jsp
@@ -0,0 +1,21 @@
+<%--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+--%>
+<JSP>add JSP CODE here </jsp>
\ No newline at end of file