You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2019/01/23 10:27:08 UTC
svn commit: r1851902 - in /jackrabbit/oak/branches/1.8: ./
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/
oak-segment-tar/src/test/java/org...
Author: frm
Date: Wed Jan 23 10:27:08 2019
New Revision: 1851902
URL: http://svn.apache.org/viewvc?rev=1851902&view=rev
Log:
OAK-6749 - Don't trigger a binary download request for in-memory Blob IDs
Added:
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobFetchTimeoutException.java
- copied unchanged from r1851533, jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobFetchTimeoutException.java
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobProcessor.java
- copied unchanged from r1851533, jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobProcessor.java
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobTypeUnknownException.java
- copied unchanged from r1851533, jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobTypeUnknownException.java
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobWriteException.java
- copied unchanged from r1851533, jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobWriteException.java
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java
- copied, changed from r1851535, jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/
- copied from r1851534, jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/
Modified:
jackrabbit/oak/branches/1.8/ (props changed)
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java
Propchange: jackrabbit/oak/branches/1.8/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jan 23 10:27:08 2019
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1820660-1820661,1820729,1820734,1820859,1820861,1820878,1820888,1820947,1821027,1821130,1821140-1821141,1821178,1821237,1821240,1821249,1821258,1821325,1821358,1821361-1821362,1821370,1821375,1821393,1821477,1821487,1821516,1821617,1821663,1821665,1821668,1821681,1821847,1821975-1821983,1822121,1822201,1822207,1822527,1822642,1822723,1822808,1822850,1822934,1823135,1823163,1823169,1823172,1823655,1823669,1824196,1824198,1824253,1824255,1824896,1824962,1825065,1825362,1825381,1825442,1825448,1825466,1825470-1825471,1825475,1825523,1825525,1825561,1825619-1825621,1825651,1825654,1825992,1826079,1826090,1826096,1826216,1826237,1826338,1826516,1826532,1826551,1826560,1826638,1826640,1826730,1826833,1826932,1826957,1827423,1827472,1827486,1827816,1827977,1828349,1828439,1828502,1828529,1828948,1829527,1829534,1829546,1829569,1829587,1829665,1829854,1829864,1829978,1829985,1829987,1829998,1830019,1830048,1830160,1830171,1830197,1830209,1830239,1830347,1830748,1830911
,1830923,1831157-1831158,1831163,1831190,1831374,1831560,1831689,1832258,1832376,1832379,1832535,1833308,1833347,1833833,1834112,1834117,1834287,1834291,1834302,1834326,1834328,1834336,1834428,1834468,1834483,1834610,1834648-1834649,1834681,1834823,1834857-1834858,1835060,1835518,1835521,1835635,1835642,1835780,1835819,1836082,1836121,1836167-1836168,1836170-1836187,1836189-1836196,1836206,1836487,1836493,1837057,1837274,1837296,1837326,1837475,1837503,1837547,1837569,1837600,1837657,1837718,1837998,1838076,1838637,1839549,1839570,1839637,1839746,1840019,1840024,1840031,1840226,1840455,1840462,1840574,1841314,1841352,1842089,1842677,1843175,1843222,1843231,1843398,1843618,1843652,1843911,1844325,1844549,1844625,1844627,1844642,1844728,1844775,1844932,1845135,1845336,1845405,1845415,1845730-1845731,1845863,1845865,1846057,1846617,1848073,1848181-1848182,1848191,1848217,1848822-1848823,1850837
+/jackrabbit/oak/trunk:1820660-1820661,1820729,1820734,1820859,1820861,1820878,1820888,1820947,1821027,1821130,1821140-1821141,1821178,1821237,1821240,1821249,1821258,1821325,1821358,1821361-1821362,1821370,1821375,1821393,1821477,1821487,1821516,1821617,1821663,1821665,1821668,1821681,1821847,1821975-1821983,1822121,1822201,1822207,1822527,1822642,1822723,1822808,1822850,1822934,1823135,1823163,1823169,1823172,1823655,1823669,1824196,1824198,1824253,1824255,1824896,1824962,1825065,1825362,1825381,1825442,1825448,1825466,1825470-1825471,1825475,1825523,1825525,1825561,1825619-1825621,1825651,1825654,1825992,1826079,1826090,1826096,1826216,1826237,1826338,1826516,1826532,1826551,1826560,1826638,1826640,1826730,1826833,1826932,1826957,1827423,1827472,1827486,1827816,1827977,1828349,1828439,1828502,1828529,1828948,1829527,1829534,1829546,1829569,1829587,1829665,1829854,1829864,1829978,1829985,1829987,1829998,1830019,1830048,1830160,1830171,1830197,1830209,1830239,1830347,1830748,1830911
,1830923,1831157-1831158,1831163,1831190,1831374,1831560,1831689,1832258,1832376,1832379,1832535,1833308,1833347,1833833,1834112,1834117,1834287,1834291,1834302,1834326,1834328,1834336,1834428,1834468,1834483,1834610,1834648-1834649,1834681,1834823,1834857-1834858,1835060,1835518,1835521,1835635,1835642,1835780,1835819,1836082,1836121,1836167-1836168,1836170-1836187,1836189-1836196,1836206,1836487,1836493,1837057,1837274,1837296,1837326,1837475,1837503,1837547,1837569,1837600,1837657,1837718,1837998,1838076,1838637,1839549,1839570,1839637,1839746,1840019,1840024,1840031,1840226,1840455,1840462,1840574,1841314,1841352,1842089,1842677,1843175,1843222,1843231,1843398,1843618,1843652,1843911,1844325,1844549,1844625,1844627,1844642,1844728,1844775,1844932,1845135,1845336,1845405,1845415,1845730-1845731,1845863,1845865,1846057,1846617,1848073,1848181-1848182,1848191,1848217,1848822-1848823,1850837,1851533-1851535,1851619
/jackrabbit/trunk:1345480
Copied: jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java (from r1851535, jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java?p2=jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java&p1=jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java&r1=1851535&r2=1851902&rev=1851902&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java (original)
+++ jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java Wed Jan 23 10:27:08 2019
@@ -19,6 +19,8 @@
package org.apache.jackrabbit.oak.segment.standby.client;
+import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly;
+
import java.io.IOException;
import java.io.InputStream;
@@ -59,7 +61,59 @@ class RemoteBlobProcessor implements Blo
}
private boolean shouldFetchBinary(SegmentBlob blob) {
- return blob.isExternal() && blob.getReference() == null && blob.getBlobId() != null;
+
+ // Shortcut: If the Blob ID is null, this is an inline binary and we
+ // don't have to fetch it.
+
+ String blobId = blob.getBlobId();
+
+ if (blobId == null) {
+ return false;
+ }
+
+ // Shortcut: If the Blob Store is able to retrieve a non-null reference
+ // to the Blob, we can be sure that the Blob is already stored locally.
+ // We don't have to download it.
+
+ String reference;
+
+ try {
+ reference = blob.getReference();
+ } catch (Exception e) {
+ reference = null;
+ }
+
+ if (reference != null) {
+ return false;
+ }
+
+ // Worst case: A null reference to the Blob might just mean that the
+ // Blob Store doesn't support references. The Blob might still be stored
+ // locally. We have to retrieve an InputStream for the Blob, and
+ // perform a tentative read in order to overcome a possible lazy
+ // implementation of the returned InputStream.
+
+ InputStream data;
+
+ try {
+ data = blobStore.getInputStream(blobId);
+ } catch (Exception e) {
+ return true;
+ }
+
+ if (data == null) {
+ return true;
+ }
+
+ try {
+ data.read();
+ } catch (Exception e) {
+ return true;
+ } finally {
+ closeQuietly(data);
+ }
+
+ return false;
}
private void fetchAndStoreBlob(String blobId) throws InterruptedException {
Modified: jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java?rev=1851902&r1=1851901&r2=1851902&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java (original)
+++ jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java Wed Jan 23 10:27:08 2019
@@ -31,7 +31,6 @@ import org.apache.jackrabbit.oak.api.Blo
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.segment.CancelableDiff;
-import org.apache.jackrabbit.oak.segment.SegmentBlob;
import org.apache.jackrabbit.oak.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
@@ -51,24 +50,30 @@ class StandbyDiff implements NodeStateDi
private final StandbyClient client;
- private final boolean hasDataStore;
-
private final String path;
private final Supplier<Boolean> running;
+ private final BlobProcessor blobProcessor;
+
StandbyDiff(NodeBuilder builder, FileStore store, StandbyClient client, Supplier<Boolean> running) {
this(builder, store, client, "/", running);
}
- private StandbyDiff(NodeBuilder builder, FileStore store, StandbyClient client, String path,
- Supplier<Boolean> running) {
+ private static BlobProcessor newBinaryFetcher(BlobStore blobStore, StandbyClient client) {
+ if (blobStore == null) {
+ return (blob) -> {};
+ }
+ return new RemoteBlobProcessor(blobStore, client::getBlob);
+ }
+
+ private StandbyDiff(NodeBuilder builder, FileStore store, StandbyClient client, String path, Supplier<Boolean> running) {
this.builder = builder;
this.store = store;
- this.hasDataStore = store.getBlobStore() != null;
this.client = client;
this.path = path;
this.running = running;
+ this.blobProcessor = newBinaryFetcher(store.getBlobStore(), client);
}
@Override
@@ -129,7 +134,7 @@ class StandbyDiff implements NodeStateDi
return (SegmentNodeState) after;
}
- if (!hasDataStore) {
+ if (store.getBlobStore() == null) {
return (SegmentNodeState) after;
}
@@ -137,7 +142,7 @@ class StandbyDiff implements NodeStateDi
// traversal to verify binaries
for (PropertyState propertyState : after.getProperties()) {
- fetchBinary(propertyState);
+ processBinary(propertyState);
}
boolean success = after.compareAgainstBaseState(before, new CancelableDiff(this, newCanceledSupplier()));
@@ -162,59 +167,44 @@ class StandbyDiff implements NodeStateDi
};
}
- private PropertyState fetchBinary(PropertyState property) {
+ private PropertyState processBinary(PropertyState property) {
Type<?> type = property.getType();
if (type == BINARY) {
- fetchBinary(property.getValue(Type.BINARY), property.getName());
+ processBinary(property.getValue(Type.BINARY), property.getName());
} else if (type == BINARIES) {
for (Blob blob : property.getValue(BINARIES)) {
- fetchBinary(blob, property.getName());
+ processBinary(blob, property.getName());
}
}
return property;
}
- private void fetchBinary(Blob b, String pName) {
- if (b instanceof SegmentBlob) {
- fetchBinary((SegmentBlob) b, pName);
- } else {
- log.warn("Unknown Blob {} at {}, ignoring", b.getClass().getName(), path + "#" + pName);
- }
- }
-
- private void fetchBinary(SegmentBlob sb, String pName) {
- if (sb.isExternal() && hasDataStore && sb.getReference() == null) {
- String blobId = sb.getBlobId();
-
- if (blobId == null) {
- return;
- }
-
- try {
- fetchAndStoreBlob(blobId, pName);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
- }
-
- private void fetchAndStoreBlob(String blobId, String pName) throws InterruptedException {
- InputStream in = client.getBlob(blobId);
-
- if (in == null) {
- throw new IllegalStateException("Unable to load remote blob " + blobId + " at " + path + "#" + pName
- + " in " + client.getReadTimeoutMs() + "ms. Please increase the timeout and try again.");
- }
-
+ private void processBinary(Blob b, String propertyName) {
try {
- BlobStore blobStore = store.getBlobStore();
- assert blobStore != null : "Blob store must not be null";
- blobStore.writeBlob(in);
- in.close();
- } catch (IOException f) {
- throw new IllegalStateException("Unable to persist blob " + blobId + " at " + path + "#" + pName, f);
+ blobProcessor.processBinary(b);
+ } catch (BlobFetchTimeoutException e) {
+ String message = String.format(
+ "Unable to load remote blob %s at %s#%s in %dms. Please increase the timeout and try again.",
+ e.getBlobId(),
+ path,
+ propertyName,
+ client.getReadTimeoutMs()
+ );
+ throw new IllegalStateException(message, e);
+ } catch (BlobWriteException e) {
+ String message = String.format(
+ "Unable to persist blob %s at %s#%s",
+ e.getBlobId(),
+ path,
+ propertyName
+ );
+ throw new IllegalStateException(message, e);
+ } catch (BlobTypeUnknownException e) {
+ log.warn("Unknown Blob {} at {}, ignoring", b.getClass().getName(), path + "#" + propertyName);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
}
}
Modified: jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java?rev=1851902&r1=1851534&r2=1851902&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java (original)
+++ jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java Wed Jan 23 10:27:08 2019
@@ -33,7 +33,6 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.junit.Assert;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.RuleChain;
@@ -85,7 +84,6 @@ public class RemoteBlobProcessorTest {
* downloaded.
*/
@Test
- @Ignore("OAK-6749")
public void inMemoryBinaryShouldNotBeDownloaded() throws Exception {
SegmentNodeStore store = SegmentNodeStoreBuilders.builder(fileStore.fileStore()).build();
Modified: jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java?rev=1851902&r1=1851901&r2=1851902&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java (original)
+++ jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java Wed Jan 23 10:27:08 2019
@@ -37,11 +37,15 @@ public class TemporaryBlobStore extends
@Override
protected void before() throws Throwable {
FileDataStore fds = new FileDataStore();
- fds.setMinRecordLength(4092);
+ configureDataStore(fds);
fds.init(folder.newFolder().getAbsolutePath());
store = new DataStoreBlobStore(fds);
}
+ protected void configureDataStore(FileDataStore dataStore) {
+ dataStore.setMinRecordLength(4092);
+ }
+
@Override
protected void after() {
try {