You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by re...@apache.org on 2014/05/23 17:24:48 UTC
svn commit: r1597115 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java
Author: reschke
Date: Fri May 23 15:24:47 2014
New Revision: 1597115
URL: http://svn.apache.org/r1597115
Log:
OAK-1266 - allow configuring the number of documents to be created in one transaction, add related test case
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java?rev=1597115&r1=1597114&r2=1597115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java Fri May 23 15:24:47 2014
@@ -398,15 +398,23 @@ public class RDBDocumentStore implements
@CheckForNull
private <T extends Document> boolean internalCreate(Collection<T> collection, List<UpdateOp> updates) {
try {
- for (UpdateOp update : updates) {
- T doc = collection.newDocument(this);
- update.increment(MODCOUNT, 1);
- UpdateUtils.applyChanges(doc, update, comparator);
- if (!update.getId().equals(doc.getId())) {
- throw new MicroKernelException("ID mismatch - UpdateOp: " + update.getId() + ", ID property: " + doc.getId());
+ // try up to CHUNKSIZE ops in one transaction
+ for (List<UpdateOp> chunks : Lists.partition(updates, CHUNKSIZE)) {
+ List<T> docs = new ArrayList<T>();
+ for (UpdateOp update : chunks) {
+ T doc = collection.newDocument(this);
+ update.increment(MODCOUNT, 1);
+ UpdateUtils.applyChanges(doc, update, comparator);
+ if (!update.getId().equals(doc.getId())) {
+ throw new MicroKernelException("ID mismatch - UpdateOp: " + update.getId() + ", ID property: "
+ + doc.getId());
+ }
+ docs.add(doc);
+ }
+ insertDocuments(collection, docs);
+ for (T doc : docs) {
+ addToCache(collection, doc);
}
- insertDocument(collection, doc);
- addToCache(collection, doc);
}
return true;
} catch (MicroKernelException ex) {
@@ -432,7 +440,7 @@ public class RDBDocumentStore implements
update.increment(MODCOUNT, 1);
UpdateUtils.applyChanges(doc, update, comparator);
try {
- insertDocument(collection, doc);
+ insertDocuments(collection, Collections.singletonList(doc));
addToCache(collection, doc);
return oldDoc;
} catch (MicroKernelException ex) {
@@ -668,20 +676,23 @@ public class RDBDocumentStore implements
}
}
- private <T extends Document> void insertDocument(Collection<T> collection, T document) {
+ private <T extends Document> void insertDocuments(Collection<T> collection, List<T> documents) {
Connection connection = null;
String tableName = getTable(collection);
+ List<String> ids = new ArrayList<String>();
try {
connection = getConnection();
- String data = asString(document);
- Long modified = (Long) document.get(MODIFIED);
- Number flag = (Number) document.get(NodeDocument.HAS_BINARY_FLAG);
- Boolean hasBinary = flag == null ? false : flag.intValue() == NodeDocument.HAS_BINARY_VAL;
- Long modcount = (Long) document.get(MODCOUNT);
- dbInsert(connection, tableName, document.getId(), modified, hasBinary, modcount, data);
+ for (T document : documents) {
+ String data = asString(document);
+ Long modified = (Long) document.get(MODIFIED);
+ Number flag = (Number) document.get(NodeDocument.HAS_BINARY_FLAG);
+ Boolean hasBinary = flag == null ? false : flag.intValue() == NodeDocument.HAS_BINARY_VAL;
+ Long modcount = (Long) document.get(MODCOUNT);
+ dbInsert(connection, tableName, document.getId(), modified, hasBinary, modcount, data);
+ }
connection.commit();
} catch (SQLException ex) {
- LOG.debug("insert of " + document.getId() + " failed", ex);
+ LOG.debug("insert of " + ids + " failed", ex);
try {
if (connection != null) {
connection.rollback();
@@ -695,10 +706,16 @@ public class RDBDocumentStore implements
}
}
+ // configuration
+
+ // Whether to use GZIP compression
+ private static boolean NOGZIP = Boolean.getBoolean("org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.NOGZIP");
+ // Number of documents to insert at once for batch create
+ private static int CHUNKSIZE = Integer.getInteger("org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.CHUNKSIZE", 64);
+
// low level operations
private static byte[] GZIPSIG = { 31, -117 };
- private static boolean NOGZIP = Boolean.getBoolean("org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.NOGZIP");
private String getData(ResultSet rs, int stringIndex, int blobIndex) throws SQLException {
try {
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java?rev=1597115&r1=1597114&r2=1597115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java Fri May 23 15:24:47 2014
@@ -261,32 +261,43 @@ public class BasicDocumentStoreTest exte
@Test
public void testCreatePerfSmall() {
- createPerf(16);
+ createPerf(16, 1);
+ }
+
+ @Test
+ public void testCreatePerfSmallBatch() {
+ createPerf(16, 64);
}
@Test
public void testCreatePerfBig() {
- createPerf(32 * 1024);
+ createPerf(32 * 1024, 1);
}
- private void createPerf(int size) {
+ private void createPerf(int size, int amount) {
String pval = generateString(size);
long duration = 1000;
long end = System.currentTimeMillis() + duration;
long cnt = 0;
+ List<String> ids = new ArrayList<String>();
while (System.currentTimeMillis() < end) {
- String id = this.getClass().getName() + ".testCreatePerf-" + size + "-" + cnt;
- UpdateOp up = new UpdateOp(id, true);
- up.set("_id", id);
- up.set("foo", pval);
- boolean success = super.ds.create(Collection.NODES, Collections.singletonList(up));
- assertTrue("document with " + id + " not created", success);
- removeMe.add(id);
+ List<UpdateOp> ups = new ArrayList<UpdateOp>();
+ for (int i = 0; i < amount; i++) {
+ String id = this.getClass().getName() + ".testCreatePerf-" + size + "-" + cnt + "-" + i;
+ UpdateOp up = new UpdateOp(id, true);
+ up.set("_id", id);
+ up.set("foo", pval);
+ ups.add(up);
+ ids.add(id);
+ }
+ boolean success = super.ds.create(Collection.NODES, ups);
+ removeMe.addAll(ids);
+ assertTrue("documents with " + ids + " not created", success);
cnt += 1;
}
- LOG.info("document creation with property of size " + size + " for " + super.dsname + " was " + cnt + " in " + duration + "ms ("
+ LOG.info("document creation with property of size " + size + " and batch size " + amount + " for " + super.dsname + " was " + cnt + " in " + duration + "ms ("
+ (cnt / (duration / 1000f)) + "/s)");
}