You are viewing a plain text version of this content. The canonical link for it is here.
Posted to httpclient-users@hc.apache.org by "Lansing, Carina S" <Ca...@pnnl.gov> on 2015/01/27 03:46:40 UTC
Upload Performance
Hi All,
We recently upgraded from commons-httpclient (3.1) to httpcomponents-client (4.3.6) in order to support proxies with NTLM authentication. We also converted our custom upload servlet into a mutli-part form upload. After these changes, we've noticed that upload times for very large files have significantly increased. For example, a 21GB file used to take ~12 minutes to upload, now it takes ~34 minutes. Our application is for scientific data, so datasets with very large files and/or numbers of files are common (although usually the latter). I have attached our abstract web service client class which includes how we are configuring our http client, connection manager, proxies, ssl, authentication, etc. Below is an excerpt that performs the multi-part call. Please let me know if something stands out that is a blatant API no-no, as I may have missed something from the documentation and mailing lists. If not, do you have any suggestions on the best approach to remedy this problem? Has anyone encountered this same issue?
Many Thanks!
Carina
private class MultipartFormUploadRunnable implements Runnable {
private Map<File, CmsPath> fileUploads;
private IProgressMonitor monitor;
private int totalFiles;
private int workedByteBatchSize;
private File globalMetadataFile;
private Map<String, File> fileSpecificMetadataFiles;
public MultipartFormUploadRunnable(Map<File, CmsPath> fileUploads, File globalMetadataFile,
Map<String, File> fileSpecificMetadataFiles, IProgressMonitor monitor, int totalFiles, int workedByteBatchSize) {
super();
this.fileUploads = fileUploads;
this.monitor = monitor;
this.totalFiles = totalFiles;
this.workedByteBatchSize = workedByteBatchSize;
this.globalMetadataFile = globalMetadataFile;
this.fileSpecificMetadataFiles = fileSpecificMetadataFiles;
}
/* (non-Javadoc)
* @see java.lang.Thread#run()
*/
@Override
public void run() {
StringBuilder url = getCatWebScriptUrl();
WebServiceUrlUtility.appendPaths(url, "upload");
WebServiceUrlUtility.appendParameter(url, "enableNotififcations", "true");
WebServiceUrlUtility.appendParameter(url, "enablePipeline", "true");
WebServiceUrlUtility.appendParameter(url, "batchSize", "20");
String encodedUrl = WebServiceUrlUtility.encode(url);
CloseableHttpResponse response = null;
try {
MultipartEntityBuilder reqBuilder = MultipartEntityBuilder.create();
for(File fileToUpload : fileUploads.keySet()) {
// File form field has path for key
// where path = path on server where the file will be located
CmsPath filePath = fileUploads.get(fileToUpload);
FileBody file = new FileBody(fileToUpload);
reqBuilder = reqBuilder.addPart(filePath.toAssociationNamePath(), file);
}
// Then create global metadata file form field
if(globalMetadataFile != null) {
FileBody globalMetadata = new FileBody(globalMetadataFile);
reqBuilder = reqBuilder.addPart("globalMetadataFile", globalMetadata);
}
// then create file-specific metadata file
if(fileSpecificMetadataFiles != null) {
for(String regex : fileSpecificMetadataFiles.keySet()) {
File metadataFile = fileSpecificMetadataFiles.get(regex);
FileBody fileBody = new FileBody(metadataFile);
String fileKey = "metadataFile_" + regex;
reqBuilder = reqBuilder.addPart(fileKey, fileBody);
}
}
HttpEntity wrappedEntity = reqBuilder.build();
// decorate http entity with progress monitor
HttpEntity reqEntity = new HttpEntityWithProgress(wrappedEntity, monitor, workedByteBatchSize);
HttpPost httppost = new HttpPost(encodedUrl);
httppost.setEntity((HttpEntity)reqEntity);
response = executeMethod(httpClient, httppost);
EntityUtils.consumeQuietly(response.getEntity());
} catch (Throwable e) {
handleException("Failed to execute method.", e);
} finally {
closeQuietly(response);
}
}
}
Re: Upload Performance
Posted by Stefan Magnus Landrø <st...@gmail.com>.
Has the buffer size been changed on either client or server?
> Den 27. jan. 2015 kl. 03.46 skrev Lansing, Carina S <Ca...@pnnl.gov>:
>
> Hi All,
>
> We recently upgraded from commons-httpclient (3.1) to httpcomponents-client (4.3.6) in order to support proxies with NTLM authentication. We also converted our custom upload servlet into a mutli-part form upload. After these changes, we’ve noticed that upload times for very large files have significantly increased. For example, a 21GB file used to take ~12 minutes to upload, now it takes ~34 minutes. Our application is for scientific data, so datasets with very large files and/or numbers of files are common (although usually the latter). I have attached our abstract web service client class which includes how we are configuring our http client, connection manager, proxies, ssl, authentication, etc. Below is an excerpt that performs the multi-part call. Please let me know if something stands out that is a blatant API no-no, as I may have missed something from the documentation and mailing lists. If not, do you have any suggestions on the best approach to remedy this problem? Has anyone encountered this same issue?
>
> Many Thanks!
> Carina
>
>
> private class MultipartFormUploadRunnable implements Runnable {
> private Map<File, CmsPath> fileUploads;
> private IProgressMonitor monitor;
> private int totalFiles;
> private int workedByteBatchSize;
> private File globalMetadataFile;
> private Map<String, File> fileSpecificMetadataFiles;
>
>
> public MultipartFormUploadRunnable(Map<File, CmsPath> fileUploads, File globalMetadataFile,
> Map<String, File> fileSpecificMetadataFiles, IProgressMonitor monitor, int totalFiles, int workedByteBatchSize) {
> super();
> this.fileUploads = fileUploads;
> this.monitor = monitor;
> this.totalFiles = totalFiles;
> this.workedByteBatchSize = workedByteBatchSize;
> this.globalMetadataFile = globalMetadataFile;
> this.fileSpecificMetadataFiles = fileSpecificMetadataFiles;
> }
>
> /* (non-Javadoc)
> * @see java.lang.Thread#run()
> */
> @Override
> public void run() {
> StringBuilder url = getCatWebScriptUrl();
> WebServiceUrlUtility.appendPaths(url, "upload");
> WebServiceUrlUtility.appendParameter(url, "enableNotififcations", "true");
> WebServiceUrlUtility.appendParameter(url, "enablePipeline", "true");
> WebServiceUrlUtility.appendParameter(url, "batchSize", "20");
> String encodedUrl = WebServiceUrlUtility.encode(url);
>
>
> CloseableHttpResponse response = null;
>
> try {
>
> MultipartEntityBuilder reqBuilder = MultipartEntityBuilder.create();
> for(File fileToUpload : fileUploads.keySet()) {
> // File form field has path for key
> // where path = path on server where the file will be located
> CmsPath filePath = fileUploads.get(fileToUpload);
> FileBody file = new FileBody(fileToUpload);
> reqBuilder = reqBuilder.addPart(filePath.toAssociationNamePath(), file);
> }
>
> // Then create global metadata file form field
> if(globalMetadataFile != null) {
> FileBody globalMetadata = new FileBody(globalMetadataFile);
> reqBuilder = reqBuilder.addPart("globalMetadataFile", globalMetadata);
> }
>
> // then create file-specific metadata file
> if(fileSpecificMetadataFiles != null) {
> for(String regex : fileSpecificMetadataFiles.keySet()) {
> File metadataFile = fileSpecificMetadataFiles.get(regex);
> FileBody fileBody = new FileBody(metadataFile);
>
> String fileKey = "metadataFile_" + regex;
> reqBuilder = reqBuilder.addPart(fileKey, fileBody);
> }
> }
>
> HttpEntity wrappedEntity = reqBuilder.build();
> // decorate http entity with progress monitor
> HttpEntity reqEntity = new HttpEntityWithProgress(wrappedEntity, monitor, workedByteBatchSize);
>
> HttpPost httppost = new HttpPost(encodedUrl);
> httppost.setEntity((HttpEntity)reqEntity);
> response = executeMethod(httpClient, httppost);
> EntityUtils.consumeQuietly(response.getEntity());
>
> } catch (Throwable e) {
> handleException("Failed to execute method.", e);
> } finally {
> closeQuietly(response);
> }
> }
>
> }
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: httpclient-users-unsubscribe@hc.apache.org
> For additional commands, e-mail: httpclient-users-help@hc.apache.org