You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by re...@apache.org on 2014/04/03 22:34:55 UTC
svn commit: r1584385 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/document/rdb/
test/java/org/apache/jackrabbit/oak/plugins/document/
Author: reschke
Date: Thu Apr 3 20:34:55 2014
New Revision: 1584385
URL: http://svn.apache.org/r1584385
Log:
OAK-1590 - add very basic performance and concurrency tests; add retries for crate/update race conditions (WIP)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentStoreTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentStoreFixture.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java?rev=1584385&r1=1584384&r2=1584385&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java Thu Apr 3 20:34:55 2014
@@ -155,8 +155,8 @@ public class RDBDocumentStore implements
@Override
public <T extends Document> void remove(Collection<T> collection, List<String> keys) {
- //TODO Use batch delete
- for(String key : keys){
+ // TODO Use batch delete
+ for (String key : keys) {
remove(collection, key);
}
}
@@ -234,6 +234,9 @@ public class RDBDocumentStore implements
// string length at which we switch to BLOB storage
private static int DATALIMIT = 16384 / 4;
+ // number of retries for updates
+ private static int RETRIES = 10;
+
private void initialize(DataSource ds, DocumentMK.Builder builder) throws Exception {
this.ds = ds;
@@ -247,13 +250,12 @@ public class RDBDocumentStore implements
try {
con.setAutoCommit(false);
- for (String tableName : new String[] { "CLUSTERNODES", "NODES", "SETTINGS"}) {
+ for (String tableName : new String[] { "CLUSTERNODES", "NODES", "SETTINGS" }) {
try {
PreparedStatement stmt = con.prepareStatement("select ID from " + tableName + " where ID = ?");
stmt.setString(1, "0:/");
stmt.executeQuery();
- }
- catch(SQLException ex) {
+ } catch (SQLException ex) {
// table does not appear to exist
con.rollback();
@@ -262,17 +264,19 @@ public class RDBDocumentStore implements
Statement stmt = con.createStatement();
- // the code below likely will need to be extended for new database types
+ // the code below likely will need to be extended for new
+ // database types
if ("PostgreSQL".equals(dbtype)) {
- stmt.execute("create table " + tableName
+ stmt.execute("create table "
+ + tableName
+ " (ID varchar(1000) not null primary key, MODIFIED bigint, MODCOUNT bigint, SIZE bigint, DATA varchar(16384), BDATA bytea)");
- }
- else if ("DB2".equals(dbtype) || (dbtype != null && dbtype.startsWith("DB2/"))) {
- stmt.execute("create table " + tableName
+ } else if ("DB2".equals(dbtype) || (dbtype != null && dbtype.startsWith("DB2/"))) {
+ stmt.execute("create table "
+ + tableName
+ " (ID varchar(1000) not null primary key, MODIFIED bigint, MODCOUNT bigint, SIZE bigint, DATA varchar(16384), BDATA blob)");
- }
- else {
- stmt.execute("create table " + tableName
+ } else {
+ stmt.execute("create table "
+ + tableName
+ " (ID varchar(1000) not null primary key, MODIFIED bigint, MODCOUNT bigint, SIZE bigint, DATA varchar(16384), BDATA blob)");
}
stmt.close();
@@ -325,15 +329,32 @@ public class RDBDocumentStore implements
}
update.increment(MODCOUNT, 1);
UpdateUtils.applyChanges(doc, update, comparator);
- insertDocument(collection, doc);
- addToCache(collection, doc);
- } else {
- T doc = applyChanges(collection, oldDoc, update, checkConditions);
- if (doc == null) {
- return null;
+ try {
+ insertDocument(collection, doc);
+ addToCache(collection, doc);
+ return oldDoc;
}
+ catch (MicroKernelException ex) {
+ // may have failed due to a race condition; try update instead
+ oldDoc = readDocument(collection, update.getId());
+ if (oldDoc == null) {
+ // something else went wrong
+ throw(ex);
+ }
+ return internalUpdate(collection, update, oldDoc, checkConditions, RETRIES);
+ }
+ } else {
+ return internalUpdate(collection, update, oldDoc, checkConditions, RETRIES);
+ }
+ }
- int retries = 5; // TODO
+ @CheckForNull
+ private <T extends Document> T internalUpdate(Collection<T> collection, UpdateOp update, T oldDoc, boolean checkConditions,
+ int retries) {
+ T doc = applyChanges(collection, oldDoc, update, checkConditions);
+ if (doc == null) {
+ return null;
+ } else {
boolean success = false;
while (!success && retries > 0) {
@@ -354,9 +375,9 @@ public class RDBDocumentStore implements
if (!success) {
throw new MicroKernelException("failed update (race?)");
}
- }
- return oldDoc;
+ return oldDoc;
+ }
}
@CheckForNull
@@ -541,6 +562,7 @@ public class RDBDocumentStore implements
dbInsert(connection, tableName, document.getId(), modified, modcount, data);
connection.commit();
} catch (SQLException ex) {
+ LOG.debug("insert of " + document.getId() + " failed", ex);
try {
connection.rollback();
} catch (SQLException e) {
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentStoreTest.java?rev=1584385&r1=1584384&r2=1584385&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/AbstractDocumentStoreTest.java Thu Apr 3 20:34:55 2014
@@ -18,7 +18,10 @@ package org.apache.jackrabbit.oak.plugin
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import org.junit.After;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -27,12 +30,24 @@ public abstract class AbstractDocumentSt
protected String dsname;
protected DocumentStore ds;
+ protected Set<String> removeMe = new HashSet<String>();
public AbstractDocumentStoreTest(DocumentStoreFixture dsf) {
this.ds = dsf.createDocumentStore();
this.dsname = dsf.getName();
}
+ @After
+ public void cleanUp() {
+ for (String id : removeMe) {
+ try {
+ ds.remove(org.apache.jackrabbit.oak.plugins.document.Collection.NODES, id);
+ } catch (Exception ex) {
+ // best effort
+ }
+ }
+ }
+
@Parameterized.Parameters
public static Collection<Object[]> fixtures() {
Collection<Object[]> result = new ArrayList<Object[]>();
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java?rev=1584385&r1=1584384&r2=1584385&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BasicDocumentStoreTest.java Thu Apr 3 20:34:55 2014
@@ -20,11 +20,8 @@ import static org.junit.Assert.assertNot
import static org.junit.Assert.assertTrue;
import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
-import org.junit.After;
import org.junit.Assume;
import org.junit.Test;
import org.slf4j.Logger;
@@ -34,23 +31,10 @@ public class BasicDocumentStoreTest exte
private static final Logger LOG = LoggerFactory.getLogger(BasicDocumentStoreTest.class);
- private Set<String> removeMe = new HashSet<String>();
-
public BasicDocumentStoreTest(DocumentStoreFixture dsf) {
super(dsf);
}
- @After
- public void cleanUp() {
- for (String id : removeMe) {
- try {
- super.ds.remove(Collection.NODES, id);
- } catch (Exception ex) {
- // best effort
- }
- }
- }
-
@Test
public void testAddAndRemove() {
String id = this.getClass().getName() + ".testAddAndRemove";
@@ -78,7 +62,7 @@ public class BasicDocumentStoreTest exte
while (max - min >= 2) {
test = (max + min) / 2;
- String id = generateId(test);
+ String id = generateString(test);
UpdateOp up = new UpdateOp(id, true);
up.set("_id", id);
boolean success = super.ds.create(Collection.NODES, Collections.singletonList(up));
@@ -105,7 +89,7 @@ public class BasicDocumentStoreTest exte
while (max - min >= 256) {
test = (max + min) / 2;
String id = this.getClass().getName() + ".testMaxProperty-" + test;
- String pval = generateId(test);
+ String pval = generateString(test);
UpdateOp up = new UpdateOp(id, true);
up.set("_id", id);
up.set("foo", pval);
@@ -124,10 +108,72 @@ public class BasicDocumentStoreTest exte
LOG.info("max prop length for " + super.dsname + " was " + test);
}
- private static String generateId(int length) {
+ @Test
+ public void testCreatePerfSmall() {
+ createPerf(16);
+ }
+
+ @Test
+ public void testCreatePerfBig() {
+ createPerf(32 * 1024);
+ }
+
+ private void createPerf(int size) {
+ String pval = generateString(size);
+ long duration = 1000;
+ long end = System.currentTimeMillis() + duration;
+ long cnt = 0;
+
+ while (System.currentTimeMillis() < end) {
+ String id = this.getClass().getName() + ".testCreatePerf-" + size + "-" + cnt;
+ UpdateOp up = new UpdateOp(id, true);
+ up.set("_id", id);
+ up.set("foo", pval);
+ boolean success = super.ds.create(Collection.NODES, Collections.singletonList(up));
+ assertTrue("document with " + id + " nit created", success);
+ removeMe.add(id);
+ cnt += 1;
+ }
+
+ LOG.info("document creation with property of size " + size + " for " + super.dsname + " was " + cnt + " in " + duration + "ms ("
+ + (cnt / (duration / 1000f)) + "/s)");
+ }
+
+ @Test
+ public void testUpdatePerfSmall() {
+ updatePerf(16);
+ }
+
+ @Test
+ public void testUpdatePerfBig() {
+ updatePerf(32 * 1024);
+ }
+
+ private void updatePerf(int size) {
+ String pval = generateString(size);
+ long duration = 1000;
+ long end = System.currentTimeMillis() + duration;
+ long cnt = 0;
+
+ String id = this.getClass().getName() + ".testUpdatePerf-" + size;
+ removeMe.add(id);
+
+ while (System.currentTimeMillis() < end) {
+ UpdateOp up = new UpdateOp(id, true);
+ up.set("_id", id);
+ up.set("foo", pval);
+ super.ds.createOrUpdate(Collection.NODES, up);
+ cnt += 1;
+ }
+
+ LOG.info("document updates with property of size " + size + " for " + super.dsname + " was " + cnt + " in " + duration + "ms ("
+ + (cnt / (duration / 1000f)) + "/s)");
+ }
+
+ private static String generateString(int length) {
StringBuffer buf = new StringBuffer(length);
while (length-- > 0) {
- buf.append('A' + ((int)(26 * Math.random())));
+ buf.append('A' + ((int) (26 * Math.random())));
}
return buf.toString();
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentStoreFixture.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentStoreFixture.java?rev=1584385&r1=1584384&r2=1584385&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentStoreFixture.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentStoreFixture.java Thu Apr 3 20:34:55 2014
@@ -34,7 +34,7 @@ public abstract class DocumentStoreFixtu
private static final Logger LOG = LoggerFactory.getLogger(DocumentStoreFixture.class);
public static final DocumentStoreFixture MEMORY = new MemoryFixture();
- public static final DocumentStoreFixture RDB_H2 = new RDBFixture("RDB-Postgres", "jdbc:h2:file:./target/ds-test", "sa", "");
+ public static final DocumentStoreFixture RDB_H2 = new RDBFixture("RDB-H2(file)", "jdbc:h2:file:./target/ds-test", "sa", "");
public static final DocumentStoreFixture RDB_PG = new RDBFixture("RDB-Postgres", "jdbc:postgresql:oak", "postgres", "geheim");
public static final DocumentStoreFixture RDB_DB2 = new RDBFixture("RDB-DB2", "jdbc:db2://localhost:50000/OAK", "oak", "geheim");
public static final DocumentStoreFixture MONGO = new MongoFixture("mongodb://localhost:27017/oak");
@@ -134,7 +134,7 @@ public abstract class DocumentStoreFixtu
return false;
}
}
-
+
@Override
public void dispose() {
try{