You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by re...@apache.org on 2016/01/07 17:46:18 UTC
svn commit: r1723584 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/document/rdb/
test/java/org/apache/jackrabbit/oak/plugins/document/blob/
Author: reschke
Date: Thu Jan 7 16:46:18 2016
New Revision: 1723584
URL: http://svn.apache.org/viewvc?rev=1723584&view=rev
Log:
OAK-3843: MS SQL doesn't support more than 2100 parameters in one request
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBJDBCTools.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java?rev=1723584&r1=1723583&r2=1723584&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java Thu Jan 7 16:46:18 2016
@@ -50,6 +50,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.Lists;
public class RDBBlobStore extends CachingBlobStore implements Closeable {
@@ -490,48 +491,45 @@ public class RDBBlobStore extends Cachin
@Override
public long countDeleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception {
long count = 0;
- // sanity check
- if (chunkIds.isEmpty()) {
- // sanity check, nothing to do
- return count;
- }
-
- Connection con = this.ch.getRWConnection();
- PreparedStatement prepMeta = null;
- PreparedStatement prepData = null;
- try {
- PreparedStatementComponent inClause = RDBJDBCTools.createInStatement("ID", chunkIds, false);
+ for (List<String> chunk : Lists.partition(chunkIds, RDBJDBCTools.MAX_IN_CLAUSE)) {
+ Connection con = this.ch.getRWConnection();
+ PreparedStatement prepMeta = null;
+ PreparedStatement prepData = null;
- StringBuilder metaStatement = new StringBuilder("delete from " + this.tnMeta + " where ")
- .append(inClause.getStatementComponent());
- StringBuilder dataStatement = new StringBuilder("delete from " + this.tnData + " where ")
- .append(inClause.getStatementComponent());
+ try {
+ PreparedStatementComponent inClause = RDBJDBCTools.createInStatement("ID", chunk, false);
- if (maxLastModifiedTime > 0) {
- metaStatement.append(" and LASTMOD <= ?");
- dataStatement.append(" and not exists(select * from " + this.tnMeta + " m where ID = m.ID and m.LASTMOD <= ?)");
- }
+ StringBuilder metaStatement = new StringBuilder("delete from " + this.tnMeta + " where ")
+ .append(inClause.getStatementComponent());
+ StringBuilder dataStatement = new StringBuilder("delete from " + this.tnData + " where ")
+ .append(inClause.getStatementComponent());
+
+ if (maxLastModifiedTime > 0) {
+ metaStatement.append(" and LASTMOD <= ?");
+ dataStatement.append(" and not exists(select * from " + this.tnMeta + " m where ID = m.ID and m.LASTMOD <= ?)");
+ }
- prepMeta = con.prepareStatement(metaStatement.toString());
- prepData = con.prepareStatement(dataStatement.toString());
+ prepMeta = con.prepareStatement(metaStatement.toString());
+ prepData = con.prepareStatement(dataStatement.toString());
- int mindex = 1, dindex = 1;
- mindex = inClause.setParameters(prepMeta, mindex);
- dindex = inClause.setParameters(prepData, dindex);
+ int mindex = 1, dindex = 1;
+ mindex = inClause.setParameters(prepMeta, mindex);
+ dindex = inClause.setParameters(prepData, dindex);
+
+ if (maxLastModifiedTime > 0) {
+ prepMeta.setLong(mindex, maxLastModifiedTime);
+ prepData.setLong(dindex, maxLastModifiedTime);
+ }
- if (maxLastModifiedTime > 0) {
- prepMeta.setLong(mindex, maxLastModifiedTime);
- prepData.setLong(dindex, maxLastModifiedTime);
+ count += prepMeta.executeUpdate();
+ prepData.execute();
+ } finally {
+ closeStatement(prepMeta);
+ closeStatement(prepData);
+ con.commit();
+ this.ch.closeConnection(con);
}
-
- count = prepMeta.executeUpdate();
- prepData.execute();
- } finally {
- closeStatement(prepMeta);
- closeStatement(prepData);
- con.commit();
- this.ch.closeConnection(con);
}
return count;
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java?rev=1723584&r1=1723583&r2=1723584&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java Thu Jan 7 16:46:18 2016
@@ -55,6 +55,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
/**
@@ -121,54 +122,64 @@ public class RDBDocumentStoreJDBC {
}
}
- public boolean batchedAppendingUpdate(Connection connection, RDBTableMetaData tmd, List<String> ids, Long modified,
+ public boolean batchedAppendingUpdate(Connection connection, RDBTableMetaData tmd, List<String> allIds, Long modified,
boolean setModifiedConditionally, String appendData) throws SQLException {
- String appendDataWithComma = "," + appendData;
- PreparedStatementComponent stringAppend = this.dbInfo.getConcatQuery(appendDataWithComma, tmd.getDataLimitInOctets());
- PreparedStatementComponent inClause = RDBJDBCTools.createInStatement("ID", ids, tmd.isIdBinary());
- StringBuilder t = new StringBuilder();
- t.append("update " + tmd.getName() + " set ");
- t.append(setModifiedConditionally ? "MODIFIED = case when ? > MODIFIED then ? else MODIFIED end, " : "MODIFIED = ?, ");
- t.append("MODCOUNT = MODCOUNT + 1, DSIZE = DSIZE + ?, ");
- t.append("DATA = " + stringAppend.getStatementComponent() + " ");
- t.append("where ").append(inClause.getStatementComponent());
- PreparedStatement stmt = connection.prepareStatement(t.toString());
- try {
- int si = 1;
- stmt.setObject(si++, modified, Types.BIGINT);
- if (setModifiedConditionally) {
+ boolean result = true;
+ for (List<String> ids : Lists.partition(allIds, RDBJDBCTools.MAX_IN_CLAUSE)) {
+ String appendDataWithComma = "," + appendData;
+ PreparedStatementComponent stringAppend = this.dbInfo.getConcatQuery(appendDataWithComma, tmd.getDataLimitInOctets());
+ PreparedStatementComponent inClause = RDBJDBCTools.createInStatement("ID", ids, tmd.isIdBinary());
+ StringBuilder t = new StringBuilder();
+ t.append("update " + tmd.getName() + " set ");
+ t.append(setModifiedConditionally ? "MODIFIED = case when ? > MODIFIED then ? else MODIFIED end, " : "MODIFIED = ?, ");
+ t.append("MODCOUNT = MODCOUNT + 1, DSIZE = DSIZE + ?, ");
+ t.append("DATA = " + stringAppend.getStatementComponent() + " ");
+ t.append("where ").append(inClause.getStatementComponent());
+ PreparedStatement stmt = connection.prepareStatement(t.toString());
+ try {
+ int si = 1;
stmt.setObject(si++, modified, Types.BIGINT);
+ if (setModifiedConditionally) {
+ stmt.setObject(si++, modified, Types.BIGINT);
+ }
+ stmt.setObject(si++, appendDataWithComma.length(), Types.BIGINT);
+ si = stringAppend.setParameters(stmt, si);
+ si = inClause.setParameters(stmt, si);
+ int count = stmt.executeUpdate();
+ if (count != ids.size()) {
+ LOG.debug("DB update failed: only " + result + " of " + ids.size() + " updated. Table: " + tmd.getName() + ", IDs:"
+ + ids);
+ result = false;
+ }
+ } finally {
+ stmt.close();
}
- stmt.setObject(si++, appendDataWithComma.length(), Types.BIGINT);
- si = stringAppend.setParameters(stmt, si);
- si = inClause.setParameters(stmt, si);
- int result = stmt.executeUpdate();
- if (result != ids.size()) {
- LOG.debug("DB update failed: only " + result + " of " + ids.size() + " updated. Table: " + tmd.getName() + ", IDs:"
- + ids);
- }
- return result == ids.size();
- } finally {
- stmt.close();
}
+ return result;
}
- public int delete(Connection connection, RDBTableMetaData tmd, List<String> ids) throws SQLException {
- PreparedStatement stmt;
- PreparedStatementComponent inClause = RDBJDBCTools.createInStatement("ID", ids, tmd.isIdBinary());
- String sql = "delete from " + tmd.getName() + " where " + inClause.getStatementComponent();
- stmt = connection.prepareStatement(sql);
+ public int delete(Connection connection, RDBTableMetaData tmd, List<String> allIds) throws SQLException {
+ int count = 0;
- try {
- inClause.setParameters(stmt, 1);
- int result = stmt.executeUpdate();
- if (result != ids.size()) {
- LOG.debug("DB delete failed for " + tmd.getName() + "/" + ids);
+ for (List<String> ids : Lists.partition(allIds, RDBJDBCTools.MAX_IN_CLAUSE)) {
+ PreparedStatement stmt;
+ PreparedStatementComponent inClause = RDBJDBCTools.createInStatement("ID", ids, tmd.isIdBinary());
+ String sql = "delete from " + tmd.getName() + " where " + inClause.getStatementComponent();
+ stmt = connection.prepareStatement(sql);
+
+ try {
+ inClause.setParameters(stmt, 1);
+ int result = stmt.executeUpdate();
+ if (result != ids.size()) {
+ LOG.debug("DB delete failed for " + tmd.getName() + "/" + ids);
+ }
+ count += result;
+ } finally {
+ stmt.close();
}
- return result;
- } finally {
- stmt.close();
}
+
+ return count;
}
public int delete(Connection connection, RDBTableMetaData tmd, Map<String, Map<Key, Condition>> toDelete)
@@ -389,26 +400,27 @@ public class RDBDocumentStoreJDBC {
}
if (!remainingDocuments.isEmpty()) {
- List<String> remainingDocumentIds = Lists.transform(remainingDocuments, idExtractor);
- PreparedStatementComponent inClause = RDBJDBCTools.createInStatement("ID", remainingDocumentIds, tmd.isIdBinary());
- StringBuilder sql = new StringBuilder("select ID from ").append(tmd.getName());
- sql.append(" where ").append(inClause.getStatementComponent());
-
Set<String> documentsWithUpdatedModcount = new HashSet<String>();
-
- PreparedStatement selectStmt = null;
- ResultSet rs = null;
- try {
- selectStmt = connection.prepareStatement(sql.toString());
- selectStmt.setPoolable(false);
- inClause.setParameters(selectStmt, 1);
- rs = selectStmt.executeQuery();
- while (rs.next()) {
- documentsWithUpdatedModcount.add(getIdFromRS(tmd, rs, 1));
+ List<String> remainingDocumentIds = Lists.transform(remainingDocuments, idExtractor);
+ for (List<String> keys : Lists.partition(remainingDocumentIds, RDBJDBCTools.MAX_IN_CLAUSE)) {
+ PreparedStatementComponent inClause = RDBJDBCTools.createInStatement("ID", keys, tmd.isIdBinary());
+ StringBuilder sql = new StringBuilder("select ID from ").append(tmd.getName());
+ sql.append(" where ").append(inClause.getStatementComponent());
+
+ PreparedStatement selectStmt = null;
+ ResultSet rs = null;
+ try {
+ selectStmt = connection.prepareStatement(sql.toString());
+ selectStmt.setPoolable(false);
+ inClause.setParameters(selectStmt, 1);
+ rs = selectStmt.executeQuery();
+ while (rs.next()) {
+ documentsWithUpdatedModcount.add(getIdFromRS(tmd, rs, 1));
+ }
+ } finally {
+ closeResultSet(rs);
+ closeStatement(selectStmt);
}
- } finally {
- closeResultSet(rs);
- closeStatement(selectStmt);
}
Iterator<T> it = remainingDocuments.iterator();
@@ -575,56 +587,55 @@ public class RDBDocumentStoreJDBC {
return result;
}
- public List<RDBRow> read(Connection connection, RDBTableMetaData tmd, Collection<String> keys) throws SQLException {
- if (keys.isEmpty()) {
- return Collections.emptyList();
- }
-
- PreparedStatementComponent inClause = RDBJDBCTools.createInStatement("ID", keys, tmd.isIdBinary());
- StringBuilder query = new StringBuilder();
- query.append("select ID, MODIFIED, MODCOUNT, CMODCOUNT, HASBINARY, DELETEDONCE, DATA, BDATA from ");
- query.append(tmd.getName());
- query.append(" where ").append(inClause.getStatementComponent());
+ public List<RDBRow> read(Connection connection, RDBTableMetaData tmd, Collection<String> allKeys) throws SQLException {
- PreparedStatement stmt = connection.prepareStatement(query.toString());
- stmt.setPoolable(false);
- try {
- inClause.setParameters(stmt, 1);
- ResultSet rs = stmt.executeQuery();
+ List<RDBRow> rows = new ArrayList<RDBRow>();
- List<RDBRow> rows = new ArrayList<RDBRow>();
- while (rs.next()) {
- int col = 1;
- String id = getIdFromRS(tmd, rs, col++);
- long modified = rs.getLong(col++);
- long modcount = rs.getLong(col++);
- long cmodcount = rs.getLong(col++);
- long hasBinary = rs.getLong(col++);
- long deletedOnce = rs.getLong(col++);
- String data = rs.getString(col++);
- byte[] bdata = rs.getBytes(col++);
- RDBRow row = new RDBRow(id, hasBinary == 1, deletedOnce == 1, modified, modcount, cmodcount, data, bdata);
- rows.add(row);
- }
-
- return rows;
- } catch (SQLException ex) {
- LOG.error("attempting to read " + keys, ex);
- // DB2 throws an SQLException for invalid keys; handle this more
- // gracefully
- if ("22001".equals(ex.getSQLState())) {
- try {
- connection.rollback();
- } catch (SQLException ex2) {
- LOG.debug("failed to rollback", ex2);
+ for (List<String> keys : Iterables.partition(allKeys, RDBJDBCTools.MAX_IN_CLAUSE)) {
+ PreparedStatementComponent inClause = RDBJDBCTools.createInStatement("ID", keys, tmd.isIdBinary());
+ StringBuilder query = new StringBuilder();
+ query.append("select ID, MODIFIED, MODCOUNT, CMODCOUNT, HASBINARY, DELETEDONCE, DATA, BDATA from ");
+ query.append(tmd.getName());
+ query.append(" where ").append(inClause.getStatementComponent());
+
+ PreparedStatement stmt = connection.prepareStatement(query.toString());
+ stmt.setPoolable(false);
+ try {
+ inClause.setParameters(stmt, 1);
+ ResultSet rs = stmt.executeQuery();
+
+ while (rs.next()) {
+ int col = 1;
+ String id = getIdFromRS(tmd, rs, col++);
+ long modified = rs.getLong(col++);
+ long modcount = rs.getLong(col++);
+ long cmodcount = rs.getLong(col++);
+ long hasBinary = rs.getLong(col++);
+ long deletedOnce = rs.getLong(col++);
+ String data = rs.getString(col++);
+ byte[] bdata = rs.getBytes(col++);
+ RDBRow row = new RDBRow(id, hasBinary == 1, deletedOnce == 1, modified, modcount, cmodcount, data, bdata);
+ rows.add(row);
+ }
+ } catch (SQLException ex) {
+ LOG.error("attempting to read " + keys, ex);
+ // DB2 throws an SQLException for invalid keys; handle this more
+ // gracefully
+ if ("22001".equals(ex.getSQLState())) {
+ try {
+ connection.rollback();
+ } catch (SQLException ex2) {
+ LOG.debug("failed to rollback", ex2);
+ }
+ return null;
+ } else {
+ throw (ex);
}
- return null;
- } else {
- throw (ex);
+ } finally {
+ stmt.close();
}
- } finally {
- stmt.close();
}
+ return rows;
}
@CheckForNull
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBJDBCTools.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBJDBCTools.java?rev=1723584&r1=1723583&r2=1723584&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBJDBCTools.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBJDBCTools.java Thu Jan 7 16:46:18 2016
@@ -329,8 +329,17 @@ public class RDBJDBCTools {
builder.append(')');
}
+ // see <https://issues.apache.org/jira/browse/OAK-3843>
+ public static final int MAX_IN_CLAUSE = Integer
+ .getInteger("org.apache.jackrabbit.oak.plugins.document.rdb.RDBJDBCTools.MAX_IN_CLAUSE", 2048);
+
public static PreparedStatementComponent createInStatement(final String fieldName, final Collection<String> values,
final boolean binary) {
+
+ if (values.size() > MAX_IN_CLAUSE) {
+ throw new IllegalArgumentException("Maximum size of IN clause allowed is " + MAX_IN_CLAUSE + ", but " + values.size() + " was requested");
+ }
+
return new PreparedStatementComponent() {
@Override
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java?rev=1723584&r1=1723583&r2=1723584&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/blob/RDBBlobStoreTest.java Thu Jan 7 16:46:18 2016
@@ -144,7 +144,7 @@ public class RDBBlobStoreTest extends Ab
@Test
public void testDeleteManyBlobs() throws Exception {
// see https://issues.apache.org/jira/browse/OAK-3807
- int count = 2000;
+ int count = 3000;
List<String> toDelete = new ArrayList<String>();
for (int i = 0; i < count; i++) {