You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@chemistry.apache.org by fm...@apache.org on 2011/11/19 18:25:50 UTC
svn commit: r1204024 - in /chemistry/opencmis/trunk:
chemistry-opencmis-server/chemistry-opencmis-server-bindings/src/main/java/org/apache/chemistry/opencmis/server/impl/atompub/
chemistry-opencmis-test/chemistry-opencmis-test-fit/src/test/java/org/apa...
Author: fmui
Date: Sat Nov 19 17:25:49 2011
New Revision: 1204024
URL: http://svn.apache.org/viewvc?rev=1204024&view=rev
Log:
CMIS-472: big content is now buffered in a temporary file (similar to the Web Services binding implementation)
Modified:
chemistry/opencmis/trunk/chemistry-opencmis-server/chemistry-opencmis-server-bindings/src/main/java/org/apache/chemistry/opencmis/server/impl/atompub/AtomEntryParser.java
chemistry/opencmis/trunk/chemistry-opencmis-test/chemistry-opencmis-test-fit/src/test/java/org/apache/chemistry/opencmis/fit/runtime/AbstractWriteObjectIT.java
Modified: chemistry/opencmis/trunk/chemistry-opencmis-server/chemistry-opencmis-server-bindings/src/main/java/org/apache/chemistry/opencmis/server/impl/atompub/AtomEntryParser.java
URL: http://svn.apache.org/viewvc/chemistry/opencmis/trunk/chemistry-opencmis-server/chemistry-opencmis-server-bindings/src/main/java/org/apache/chemistry/opencmis/server/impl/atompub/AtomEntryParser.java?rev=1204024&r1=1204023&r2=1204024&view=diff
==============================================================================
--- chemistry/opencmis/trunk/chemistry-opencmis-server/chemistry-opencmis-server-bindings/src/main/java/org/apache/chemistry/opencmis/server/impl/atompub/AtomEntryParser.java (original)
+++ chemistry/opencmis/trunk/chemistry-opencmis-server/chemistry-opencmis-server-bindings/src/main/java/org/apache/chemistry/opencmis/server/impl/atompub/AtomEntryParser.java Sat Nov 19 17:25:49 2011
@@ -18,8 +18,14 @@
*/
package org.apache.chemistry.opencmis.server.impl.atompub;
+import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -292,9 +298,9 @@ public class AtomEntryParser {
} else if (type.startsWith("text/")) {
bytes = readText(parser).getBytes("UTF-8");
} else {
- LightByteArrayOutputStream lbs = readBase64(parser);
- atomContentStream.setStream(lbs.getInputStream());
- atomContentStream.setLength(BigInteger.valueOf(lbs.getSize()));
+ ThresholdOutputStream ths = readBase64(parser);
+ atomContentStream.setStream(ths.getInputStream());
+ atomContentStream.setLength(BigInteger.valueOf(ths.getSize()));
}
if (bytes != null) {
@@ -321,9 +327,9 @@ public class AtomEntryParser {
if (TAG_MEDIATYPE.equals(name.getLocalPart())) {
cmisContentStream.setMimeType(readText(parser));
} else if (TAG_BASE64.equals(name.getLocalPart())) {
- LightByteArrayOutputStream lbs = readBase64(parser);
- cmisContentStream.setStream(lbs.getInputStream());
- cmisContentStream.setLength(BigInteger.valueOf(lbs.getSize()));
+ ThresholdOutputStream ths = readBase64(parser);
+ cmisContentStream.setStream(ths.getInputStream());
+ cmisContentStream.setLength(BigInteger.valueOf(ths.getSize()));
} else {
skip(parser);
}
@@ -376,8 +382,8 @@ public class AtomEntryParser {
/**
* Parses a tag that contains base64 encoded content.
*/
- private static LightByteArrayOutputStream readBase64(XMLStreamReader parser) throws Exception {
- LightByteArrayOutputStream bufferStream = new LightByteArrayOutputStream();
+ private static ThresholdOutputStream readBase64(XMLStreamReader parser) throws Exception {
+ ThresholdOutputStream bufferStream = new ThresholdOutputStream();
Base64.OutputStream b64stream = new Base64.OutputStream(bufferStream, Base64.DECODE);
next(parser);
@@ -400,6 +406,8 @@ public class AtomEntryParser {
}
}
+ b64stream.close();
+
next(parser);
return bufferStream;
@@ -564,114 +572,228 @@ public class AtomEntryParser {
return false;
}
- private static class LightByteArrayOutputStream extends OutputStream {
+ private static class ThresholdOutputStream extends OutputStream {
+
private static final int MAX_GROW = 10 * 1024 * 1024;
+ // TODO: make threshold configurable
+ private static final int THRESHOLD = 4 * 1024 * 1024;
private byte[] buf = null;
- private int size = 0;
+ private int bufSize = 0;
+ private long size;
+ private File tmpFile;
+ private OutputStream tmpStream;
- public LightByteArrayOutputStream() {
+ public ThresholdOutputStream() {
this(64 * 1024);
}
- public LightByteArrayOutputStream(int initSize) {
+ public ThresholdOutputStream(int initSize) {
if (initSize < 0) {
throw new IllegalArgumentException("Negative initial size: " + initSize);
}
buf = new byte[initSize];
}
- private void expand(int i) {
- if (size + i <= buf.length) {
+ private void expand(int nextBufferSize) throws IOException {
+ if (bufSize + nextBufferSize <= buf.length) {
return;
}
- int newSize = ((size + i) * 2 < MAX_GROW ? (size + i) * 2 : buf.length + i + MAX_GROW);
+ if (bufSize + nextBufferSize > THRESHOLD) {
+ if (tmpStream == null) {
+ tmpFile = File.createTempFile("opencmis", null);
+ tmpStream = new FileOutputStream(tmpFile);
+ }
+ tmpStream.write(buf, 0, bufSize);
+
+ if (buf.length != THRESHOLD) {
+ buf = new byte[THRESHOLD];
+ }
+ bufSize = 0;
+
+ return;
+ }
+
+ int newSize = ((bufSize + nextBufferSize) * 2 < MAX_GROW ? (bufSize + nextBufferSize) * 2 : buf.length
+ + nextBufferSize + MAX_GROW);
byte[] newbuf = new byte[newSize];
- System.arraycopy(buf, 0, newbuf, 0, size);
+ System.arraycopy(buf, 0, newbuf, 0, bufSize);
buf = newbuf;
}
- public int getSize() {
+ public long getSize() {
return size;
}
@Override
- public void write(byte[] buffer) {
+ public void write(byte[] buffer) throws IOException {
write(buffer, 0, buffer.length);
}
@Override
- public synchronized void write(byte[] buffer, int offset, int len) {
+ public void write(byte[] buffer, int offset, int len) throws IOException {
if (len == 0) {
return;
}
expand(len);
- System.arraycopy(buffer, offset, buf, size, len);
+ System.arraycopy(buffer, offset, buf, bufSize, len);
+ bufSize += len;
size += len;
}
@Override
- public void write(int oneByte) {
- if (size == buf.length) {
+ public void write(int oneByte) throws IOException {
+ if (bufSize == buf.length) {
expand(1);
}
- buf[size++] = (byte) oneByte;
+ buf[bufSize++] = (byte) oneByte;
+ size++;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ if (tmpStream != null) {
+ if (bufSize > 0) {
+ tmpStream.write(buf, 0, bufSize);
+ bufSize = 0;
+ }
+ tmpStream.flush();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ flush();
+
+ if (tmpStream != null) {
+ tmpStream.close();
+ }
}
- public InputStream getInputStream() {
- return new InputStream() {
+ public InputStream getInputStream() throws Exception {
+ if (tmpStream != null) {
+ close();
+ buf = null;
+
+ return new InternalTempFileInputStream();
+ } else {
+ return new InternalBufferInputStream();
+ }
+ }
+
+ private class InternalBufferInputStream extends InputStream {
+
+ private int pos = 0;
+
+ @Override
+ public boolean markSupported() {
+ return false;
+ }
+
+ @Override
+ public int available() {
+ return bufSize - pos;
+ }
+
+ @Override
+ public int read() {
+ return (pos < bufSize) && (buf != null) ? (buf[pos++] & 0xff) : -1;
+ }
- private int pos = 0;
+ @Override
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
- @Override
- public int available() {
- return size - pos;
+ @Override
+ public int read(byte[] b, int off, int len) {
+ if ((pos >= bufSize) || (buf == null)) {
+ return -1;
}
- @Override
- public int read() {
- return (pos < size) && (buf != null) ? (buf[pos++] & 0xff) : -1;
+ if ((pos + len) > bufSize) {
+ len = (bufSize - pos);
}
- @Override
- public int read(byte[] b, int off, int len) {
- if ((pos >= size) || (buf == null)) {
- return -1;
- }
+ System.arraycopy(buf, pos, b, off, len);
+ pos += len;
- if ((pos + len) > size) {
- len = (size - pos);
- }
+ return len;
+ }
- System.arraycopy(buf, pos, b, off, len);
- pos += len;
+ @Override
+ public long skip(long n) {
+ if ((pos + n) > bufSize) {
+ n = bufSize - pos;
+ }
- return len;
+ if (n < 0) {
+ return 0;
}
- @Override
- public long skip(long n) {
- if ((pos + n) > size) {
- n = size - pos;
- }
+ pos += n;
- if (n < 0) {
- return 0;
- }
+ return n;
+ }
+
+ @Override
+ public void close() throws IOException {
+ buf = null;
+ }
+ }
+
+ private class InternalTempFileInputStream extends FilterInputStream {
+
+ private boolean isDeleted = false;
- pos += n;
+ public InternalTempFileInputStream() throws FileNotFoundException {
+ super(new BufferedInputStream(new FileInputStream(tmpFile), THRESHOLD));
+ }
+
+ @Override
+ public boolean markSupported() {
+ return false;
+ }
- return n;
+ @Override
+ public int read() throws IOException {
+ int b = super.read();
+
+ if (b == -1 && !isDeleted) {
+ super.close();
+ isDeleted = tmpFile.delete();
}
- @Override
- public void close() throws IOException {
- buf = null;
+ return b;
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return read(b, 0, b.length);
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ int n = super.read(b, off, len);
+
+ if (n == -1 && !isDeleted) {
+ super.close();
+ isDeleted = tmpFile.delete();
}
- };
+
+ return n;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!isDeleted) {
+ super.close();
+ isDeleted = tmpFile.delete();
+ }
+ }
}
}
}
\ No newline at end of file
Modified: chemistry/opencmis/trunk/chemistry-opencmis-test/chemistry-opencmis-test-fit/src/test/java/org/apache/chemistry/opencmis/fit/runtime/AbstractWriteObjectIT.java
URL: http://svn.apache.org/viewvc/chemistry/opencmis/trunk/chemistry-opencmis-test/chemistry-opencmis-test-fit/src/test/java/org/apache/chemistry/opencmis/fit/runtime/AbstractWriteObjectIT.java?rev=1204024&r1=1204023&r2=1204024&view=diff
==============================================================================
--- chemistry/opencmis/trunk/chemistry-opencmis-test/chemistry-opencmis-test-fit/src/test/java/org/apache/chemistry/opencmis/fit/runtime/AbstractWriteObjectIT.java (original)
+++ chemistry/opencmis/trunk/chemistry-opencmis-test/chemistry-opencmis-test-fit/src/test/java/org/apache/chemistry/opencmis/fit/runtime/AbstractWriteObjectIT.java Sat Nov 19 17:25:49 2011
@@ -71,8 +71,8 @@ public abstract class AbstractWriteObjec
byte[] buf1 = content1.getBytes("UTF-8");
ByteArrayInputStream in1 = new ByteArrayInputStream(buf1);
- ContentStream contentStream = session.getObjectFactory().createContentStream(filename, buf1.length,
- mimetype, in1);
+ ContentStream contentStream = session.getObjectFactory().createContentStream(filename, buf1.length, mimetype,
+ in1);
assertNotNull(contentStream);
ObjectId id = session.createDocument(properties, parentId, contentStream, VersioningState.NONE);
@@ -89,8 +89,54 @@ public abstract class AbstractWriteObjec
}
@Test
- @Ignore
- public void createHugeDocument() {
+ public void createBigDocument() {
+ ObjectId parentId = session.createObjectId(fixture.getTestRootId());
+ String folderName = UUID.randomUUID().toString();
+ String typeId = FixtureData.DOCUMENT_TYPE_ID.value();
+
+ Map<String, Object> properties = new HashMap<String, Object>();
+ properties.put(PropertyIds.NAME, folderName);
+ properties.put(PropertyIds.OBJECT_TYPE_ID, typeId);
+
+ String filename = UUID.randomUUID().toString();
+ String mimetype = "application/octet-stream";
+
+ final int size = 10 * 1024 * 1024; // 10 MiB
+
+ InputStream in = new InputStream() {
+
+ private int counter = -1;
+
+ @Override
+ public int read() throws IOException {
+ counter++;
+ if (counter >= size) {
+ return -1;
+ }
+
+ return '0' + (counter / 10);
+ }
+ };
+
+ ContentStream contentStream = session.getObjectFactory().createContentStream(filename, size, mimetype, in);
+ assertNotNull(contentStream);
+
+ long start = System.currentTimeMillis();
+ ObjectId id = session.createDocument(properties, parentId, contentStream, VersioningState.NONE);
+ long end = System.currentTimeMillis();
+
+ assertNotNull(id);
+
+ Document doc = (Document) session.getObject(id);
+ assertNotNull(doc);
+
+ doc.delete(true);
+
+ log.info("createDocument with " + size + " bytes:" + (end - start) + "ms");
+ }
+
+ @Test
+ public void setBigContent() {
ObjectId parentId = session.createObjectId(fixture.getTestRootId());
String folderName = UUID.randomUUID().toString();
String typeId = FixtureData.DOCUMENT_TYPE_ID.value();
@@ -109,7 +155,7 @@ public abstract class AbstractWriteObjec
Document doc = (Document) session.getObject(id);
assertNotNull(doc);
- final int size = 1 * 1024 * 1024 * 1024; // 1GB
+ final int size = 10 * 1024 * 1024; // 10 MiB
InputStream in = new InputStream() {
@@ -133,7 +179,9 @@ public abstract class AbstractWriteObjec
doc.setContentStream(contentStream, true);
long end = System.currentTimeMillis();
- log.info("setContentStream of " + size + " bytes:" + (end - start) + "ms");
+ doc.delete(true);
+
+ log.info("setContentStream with " + size + " bytes:" + (end - start) + "ms");
}
@Test
@@ -283,8 +331,8 @@ public abstract class AbstractWriteObjec
byte[] buf1 = content1.getBytes("UTF-8");
ByteArrayInputStream in1 = new ByteArrayInputStream(buf1);
- ContentStream contentStream = session.getObjectFactory().createContentStream(filename, buf1.length,
- mimetype, in1);
+ ContentStream contentStream = session.getObjectFactory().createContentStream(filename, buf1.length, mimetype,
+ in1);
assertNotNull(contentStream);
ObjectId id = session.createDocument(properties, parentId, null, VersioningState.NONE);
@@ -320,8 +368,8 @@ public abstract class AbstractWriteObjec
byte[] buf1 = content1.getBytes("UTF-8");
ByteArrayInputStream in1 = new ByteArrayInputStream(buf1);
- ContentStream contentStream = session.getObjectFactory().createContentStream(filename, buf1.length,
- mimetype, in1);
+ ContentStream contentStream = session.getObjectFactory().createContentStream(filename, buf1.length, mimetype,
+ in1);
assertNotNull(contentStream);
ObjectId id = session.createDocument(properties, parentId, null, VersioningState.NONE);
@@ -356,8 +404,8 @@ public abstract class AbstractWriteObjec
byte[] buf1 = content1.getBytes("UTF-8");
ByteArrayInputStream in1 = new ByteArrayInputStream(buf1);
- ContentStream contentStream1 = session.getObjectFactory().createContentStream(filename1, buf1.length,
- mimetype, in1);
+ ContentStream contentStream1 = session.getObjectFactory().createContentStream(filename1, buf1.length, mimetype,
+ in1);
assertNotNull(contentStream1);
ObjectId id = session.createDocument(properties, parentId, contentStream1, VersioningState.NONE);
@@ -369,8 +417,8 @@ public abstract class AbstractWriteObjec
byte[] buf2 = content2.getBytes("UTF-8");
ByteArrayInputStream in2 = new ByteArrayInputStream(buf2);
- ContentStream contentStream2 = session.getObjectFactory().createContentStream(filename2, buf2.length,
- mimetype, in2);
+ ContentStream contentStream2 = session.getObjectFactory().createContentStream(filename2, buf2.length, mimetype,
+ in2);
assertNotNull(contentStream2);
Document doc = (Document) session.getObject(id);
@@ -402,8 +450,8 @@ public abstract class AbstractWriteObjec
byte[] buf1 = content1.getBytes("UTF-8");
ByteArrayInputStream in1 = new ByteArrayInputStream(buf1);
- ContentStream contentStream1 = session.getObjectFactory().createContentStream(filename1, buf1.length,
- mimetype, in1);
+ ContentStream contentStream1 = session.getObjectFactory().createContentStream(filename1, buf1.length, mimetype,
+ in1);
assertNotNull(contentStream1);
ObjectId id = session.createDocument(properties, parentId, contentStream1, VersioningState.NONE);
@@ -415,8 +463,8 @@ public abstract class AbstractWriteObjec
byte[] buf2 = content2.getBytes("UTF-8");
ByteArrayInputStream in2 = new ByteArrayInputStream(buf2);
- ContentStream contentStream2 = session.getObjectFactory().createContentStream(filename2, buf2.length,
- mimetype, in2);
+ ContentStream contentStream2 = session.getObjectFactory().createContentStream(filename2, buf2.length, mimetype,
+ in2);
assertNotNull(contentStream2);
Document doc = (Document) session.getObject(id);