You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2016/09/14 09:44:47 UTC
svn commit: r1760666 - in /jackrabbit/oak/branches/1.4: ./ oak-blob-cloud/
oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/
oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/stats/
oak-core/src/main/java/...
Author: amitj
Date: Wed Sep 14 09:44:47 2016
New Revision: 1760666
URL: http://svn.apache.org/viewvc?rev=1760666&view=rev
Log:
OAK-4712: Publish S3DataStore stats in JMX MBean
Merged r1760661, r1760662 from trunk
Added:
jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/stats/
- copied from r1760661, jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/stats/
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStats.java
- copied unchanged from r1760661, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStats.java
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java
- copied, changed from r1760661, jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java
- copied, changed from r1760661, jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java
jackrabbit/oak/branches/1.4/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java
- copied unchanged from r1760661, jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentS3DataStoreStatsTest.java
Modified:
jackrabbit/oak/branches/1.4/ (props changed)
jackrabbit/oak/branches/1.4/oak-blob-cloud/pom.xml
jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStore.java
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java
Propchange: jackrabbit/oak/branches/1.4/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Sep 14 09:44:47 2016
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740360,1740625-1740626,1740774,1740837,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462,1750465,1750495,1750626,1750809,1750886,1751410
,1751445-1751446,1751478,1751755,1751871,1752198,1752202,1752273-1752274,1752438,1752447,1752508,1752616,1752659,1752672,1753262,1753331-1753332,1753355,1753444,1754117,1754239,1755157,1756520,1756580,1757119,1757166,1760340,1760373,1760387
+/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740360,1740625-1740626,1740774,1740837,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462,1750465,1750495,1750626,1750809,1750886,1751410
,1751445-1751446,1751478,1751755,1751871,1752198,1752202,1752273-1752274,1752438,1752447,1752508,1752616,1752659,1752672,1753262,1753331-1753332,1753355,1753444,1754117,1754239,1755157,1756520,1756580,1757119,1757166,1760340,1760373,1760387,1760661-1760662
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.4/oak-blob-cloud/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-blob-cloud/pom.xml?rev=1760666&r1=1760665&r2=1760666&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-blob-cloud/pom.xml (original)
+++ jackrabbit/oak/branches/1.4/oak-blob-cloud/pom.xml Wed Sep 14 09:44:47 2016
@@ -41,7 +41,7 @@
<artifactId>maven-bundle-plugin</artifactId>
<configuration>
<instructions>
- <Export-Package>org.apache.jackrabbit.oak.blob.cloud.aws.s3</Export-Package>
+ <Export-Package>org.apache.jackrabbit.oak.blob.cloud.aws.s3,org.apache.jackrabbit.oak.blob.cloud.aws.s3.stats</Export-Package>
<DynamicImport-Package>sun.io</DynamicImport-Package>
</instructions>
</configuration>
@@ -101,6 +101,13 @@
<version>${jackrabbit.version}</version>
</dependency>
+ <!-- Dependencies to other Oak components -->
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-commons</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<!-- Amazon AWS dependency -->
<dependency>
<groupId>com.amazonaws</groupId>
Modified: jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStore.java?rev=1760666&r1=1760665&r2=1760666&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStore.java (original)
+++ jackrabbit/oak/branches/1.4/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3DataStore.java Wed Sep 14 09:44:47 2016
@@ -17,14 +17,26 @@
package org.apache.jackrabbit.oak.blob.cloud.aws.s3;
import java.util.Properties;
+
+import com.google.common.base.Strings;
import org.apache.jackrabbit.core.data.Backend;
import org.apache.jackrabbit.core.data.CachingDataStore;
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* An Amazon S3 data store.
*/
public class S3DataStore extends CachingDataStore {
+
+ /**
+ * Logger instance.
+ */
+ private static final Logger LOG = LoggerFactory.getLogger(S3DataStore.class);
+
protected Properties properties;
@Override
@@ -47,4 +59,24 @@ public class S3DataStore extends Caching
public void setProperties(Properties properties) {
this.properties = properties;
}
+
+ /**
+ * Look in the backend for a record matching the given identifier. Returns true
+ * if such a record exists.
+ *
+ * @param identifier - An identifier for the record.
+ * @return true if a record for the provided identifier can be found.
+ */
+ public boolean haveRecordForIdentifier(final String identifier) {
+ try {
+ if (!Strings.isNullOrEmpty(identifier)) {
+ return this.getBackend().exists(new DataIdentifier(identifier));
+ }
+ }
+ catch (DataStoreException e) {
+ LOG.warn(String.format("Data Store Exception caught checking for %s in pending uploads",
+ identifier), e);
+ }
+ return false;
+ }
}
Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java?rev=1760666&r1=1760665&r2=1760666&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreService.java Wed Sep 14 09:44:47 2016
@@ -19,18 +19,25 @@
package org.apache.jackrabbit.oak.plugins.blob.datastore;
+import java.util.Dictionary;
+import java.util.Hashtable;
import java.util.Map;
import java.util.Properties;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.ConfigurationPolicy;
import org.apache.jackrabbit.core.data.DataStore;
-import org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3DataStore;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.osgi.framework.Constants;
+import org.osgi.framework.ServiceRegistration;
import org.osgi.service.component.ComponentContext;
@Component(policy = ConfigurationPolicy.REQUIRE, name = S3DataStoreService.NAME)
public class S3DataStoreService extends AbstractDataStoreService{
public static final String NAME = "org.apache.jackrabbit.oak.plugins.blob.datastore.S3DataStore";
+ private static final String DESCRIPTION = "oak.datastore.description";
+
+ private ServiceRegistration delegateReg;
@Override
protected DataStore createDataStore(ComponentContext context, Map<String, Object> config) {
@@ -40,9 +47,26 @@ public class S3DataStoreService extends
properties.putAll(config);
dataStore.setProperties(properties);
+
+ Dictionary<String, Object> props = new Hashtable<String, Object>();
+ props.put(Constants.SERVICE_PID, dataStore.getClass().getName());
+ props.put(DESCRIPTION, getDescription());
+
+ delegateReg = context.getBundleContext().registerService(new String[] {
+ SharedS3DataStore.class.getName(),
+ SharedS3DataStore.class.getName()
+ }, dataStore , props);
+
return dataStore;
}
+ protected void deactivate() throws DataStoreException {
+ if (delegateReg != null) {
+ delegateReg.unregister();
+ }
+ super.deactivate();
+ }
+
@Override
protected String[] getDescription() {
return new String[] {"type=S3"};
Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java?rev=1760666&r1=1760665&r2=1760666&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStoreService.java Wed Sep 14 09:44:47 2016
@@ -22,14 +22,22 @@ package org.apache.jackrabbit.oak.plugin
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.ConfigurationPolicy;
import org.apache.jackrabbit.core.data.DataStore;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.osgi.framework.Constants;
+import org.osgi.framework.ServiceRegistration;
import org.osgi.service.component.ComponentContext;
+import java.util.Dictionary;
+import java.util.Hashtable;
import java.util.Map;
import java.util.Properties;
@Component(policy = ConfigurationPolicy.REQUIRE, name = SharedS3DataStoreService.NAME)
public class SharedS3DataStoreService extends AbstractDataStoreService{
public static final String NAME = "org.apache.jackrabbit.oak.plugins.blob.datastore.SharedS3DataStore";
+ private static final String DESCRIPTION = "oak.datastore.description";
+
+ private ServiceRegistration delegateReg;
@Override
protected DataStore createDataStore(ComponentContext context, Map<String, Object> config) {
@@ -39,9 +47,26 @@ public class SharedS3DataStoreService ex
properties.putAll(config);
dataStore.setProperties(properties);
+
+ Dictionary<String, Object> props = new Hashtable<String, Object>();
+ props.put(Constants.SERVICE_PID, dataStore.getClass().getName());
+ props.put(DESCRIPTION, getDescription());
+
+ delegateReg = context.getBundleContext().registerService(new String[] {
+ SharedS3DataStore.class.getName(),
+ SharedS3DataStore.class.getName()
+ }, dataStore , props);
+
return dataStore;
}
+ protected void deactivate() throws DataStoreException {
+ if (delegateReg != null) {
+ delegateReg.unregister();
+ }
+ super.deactivate();
+ }
+
@Override
protected String[] getDescription() {
return new String[] {"type=SharedS3"};
Copied: jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java (from r1760661, jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java?p2=jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java&p1=jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java&r1=1760661&r2=1760666&rev=1760666&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/S3DataStoreStatsTest.java Wed Sep 14 09:44:47 2016
@@ -20,21 +20,23 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
-import com.amazonaws.util.StringInputStream;
+import com.google.common.base.Charsets;
+import com.google.common.base.Optional;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.NullOutputStream;
import org.apache.jackrabbit.core.data.AsyncTouchCallback;
import org.apache.jackrabbit.core.data.AsyncTouchResult;
import org.apache.jackrabbit.core.data.AsyncUploadCallback;
@@ -45,9 +47,13 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.api.Blob;
-import org.apache.jackrabbit.oak.api.PropertyState;
-import org.apache.jackrabbit.oak.api.Type;
-import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.plugins.memory.MultiBinaryPropertyState;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.joda.time.DateTime;
import org.junit.After;
@@ -60,18 +66,15 @@ import static org.apache.commons.codec.b
import static org.apache.commons.io.FileUtils.copyInputStreamToFile;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class S3DataStoreStatsTest {
- @Rule public TemporaryFolder folder = new TemporaryFolder(new File("target")) {
- @Override public void delete() {
- }
- };
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
- private NodeStore mockNodeStore;
+ private static NodeStore nodeStore;
+ private static Blob mockBlob;
private static String testNodePathName = "test/node/path/name";
private File testFile;
@@ -79,48 +82,83 @@ public class S3DataStoreStatsTest {
private SharedS3DataStore defaultS3ds;
private SharedS3DataStore autoSyncMockS3ds;
private SharedS3DataStore manualSyncMockS3ds;
- private S3DataStoreStats mBean;
+ private S3DataStoreStats stats;
@Before
public void setup() throws Exception {
testFile = folder.newFile();
copyInputStreamToFile(randomStream(0, 16384), testFile);
- mockNodeStore = mock(NodeStore.class);
- MessageDigest digest = MessageDigest.getInstance("SHA-1");
- OutputStream output = new DigestOutputStream(new FileOutputStream(testFile), digest);
- FileInputStream inputStream = null;
- try {
- inputStream = new FileInputStream(testFile);
- IOUtils.copyLarge(inputStream, output);
- } finally {
- IOUtils.closeQuietly(output);
- IOUtils.closeQuietly(inputStream);
- }
- String testNodeId = encodeHexString(digest.digest());
+ String testNodeId = getIdForInputStream(new FileInputStream(testFile));
- mockNodeStore = mock(NodeStore.class);
- final NodeState mockRootState = mock(NodeState.class);
- final NodeState mockLeafState = mock(NodeState.class);
- final PropertyState mockLeafPropertyState = mock(PropertyState.class);
- final Blob mockBlob = mock(Blob.class);
- when(mockNodeStore.getRoot()).thenReturn(mockRootState);
- when(mockRootState.getChildNode(anyString())).thenReturn(mockLeafState);
- when(mockLeafState.getChildNode(anyString())).thenReturn(mockLeafState);
- when(mockLeafState.exists()).thenReturn(true);
- when(mockLeafState.getProperty(anyString())).thenReturn(mockLeafPropertyState);
- doReturn(Lists.newArrayList(mockLeafPropertyState)).when(mockLeafState).getProperties();
- doReturn(Type.BINARY).when(mockLeafPropertyState).getType();
- when(mockLeafPropertyState.getValue(Type.BINARY)).thenReturn(mockBlob);
+ mockBlob = mock(Blob.class);
when(mockBlob.getContentIdentity()).thenReturn(testNodeId);
+ nodeStore = initNodeStore(Optional.of(mockBlob),
+ Optional.<Blob>absent(),
+ Optional.<String>absent(),
+ Optional.<Integer>absent(),
+ Optional.<List<Blob>>absent());
+
defaultS3ds = mock(SharedS3DataStore.class);
- defaultS3ds.init(folder.newFolder().getAbsolutePath());
autoSyncMockS3ds = new CustomBackendS3DataStore(new TestMemoryBackend());
autoSyncMockS3ds.init(folder.newFolder().getAbsolutePath());
manualSyncMockS3ds = new CustomBackendS3DataStore(new ManuallySyncingInMemoryBackend());
manualSyncMockS3ds.init(folder.newFolder().getAbsolutePath());
}
+ private String getIdForInputStream(final InputStream in)
+ throws Exception {
+ MessageDigest digest = MessageDigest.getInstance("SHA-1");
+ OutputStream output = new DigestOutputStream(new NullOutputStream(), digest);
+ try {
+ IOUtils.copyLarge(in, output);
+ } finally {
+ IOUtils.closeQuietly(output);
+ IOUtils.closeQuietly(in);
+ }
+ return encodeHexString(digest.digest());
+ }
+
+ private static NodeStore initNodeStore(final Optional<Blob> blobProp1,
+ final Optional<Blob> blobProp2,
+ final Optional<String> stringProp,
+ final Optional<Integer> intProp,
+ final Optional<List<Blob>> blobPropList)
+ throws CommitFailedException {
+ final NodeStore nodeStore = new MemoryNodeStore();
+ NodeBuilder rootBuilder = nodeStore.getRoot().builder();
+ NodeBuilder builder = initNodeBuilder(rootBuilder);
+
+ if (blobProp1.isPresent()) {
+ builder.setProperty("blobProp1", blobProp1.get());
+ }
+ if (blobProp2.isPresent()) {
+ builder.setProperty("blobProp2", blobProp2.get());
+ }
+ if (stringProp.isPresent()) {
+ builder.setProperty("stringProp", stringProp.get());
+ }
+ if (intProp.isPresent()) {
+ builder.setProperty("intProp", intProp.get());
+ }
+ if (blobPropList.isPresent()) {
+ builder.setProperty(MultiBinaryPropertyState
+ .binaryPropertyFromBlob("blobPropList", blobPropList.get()));
+ }
+
+ nodeStore.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ return nodeStore;
+ }
+
+ private static NodeBuilder initNodeBuilder(final NodeBuilder rootBuilder) {
+ NodeBuilder builder = rootBuilder;
+ for (final String nodeName : PathUtils.elements(testNodePathName)) {
+ builder = builder.child(nodeName);
+ }
+ return builder;
+ }
+
@After
public void teardown() throws Exception {
}
@@ -128,25 +166,25 @@ public class S3DataStoreStatsTest {
@Test public void testGetActiveS3FileSyncMetricExists()
throws Exception {
- mBean = new S3DataStoreStats();
- mBean.nodeStore = mockNodeStore;
- mBean.s3ds = defaultS3ds;
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = defaultS3ds;
- assertTrue(0 == mBean.getActiveSyncs());
+ assertTrue(0 == stats.getActiveSyncs());
}
@Test
public void testGetSingleActiveS3FileSyncMetric()
throws Exception {
- mBean = new S3DataStoreStats();
- mBean.nodeStore = mockNodeStore;
- mBean.s3ds = manualSyncMockS3ds;
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = manualSyncMockS3ds;
DataRecord record = null;
try {
- record = manualSyncMockS3ds.addRecord(new StringInputStream("test"));
- assertTrue(1 == mBean.getActiveSyncs());
+ record = manualSyncMockS3ds.addRecord(getStream("test"));
+ assertTrue(1 == stats.getActiveSyncs());
} finally {
if (null != record) {
manualSyncMockS3ds.deleteRecord(record.getIdentifier());
@@ -155,24 +193,24 @@ public class S3DataStoreStatsTest {
((ManuallySyncingInMemoryBackend) manualSyncMockS3ds.getBackend()).clearInProgressWrites();
- assertTrue(0 == mBean.getActiveSyncs());
+ assertTrue(0 == stats.getActiveSyncs());
}
@Test
public void testGetMultilpleActiveS3FileSyncMetric()
throws Exception {
- mBean = new S3DataStoreStats();
- mBean.nodeStore = mockNodeStore;
- mBean.s3ds = manualSyncMockS3ds;
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = manualSyncMockS3ds;
final Set<DataRecord> records = Sets.newHashSet();
try {
- records.add(manualSyncMockS3ds.addRecord(new StringInputStream("test1")));
- records.add(manualSyncMockS3ds.addRecord(new StringInputStream("test2")));
- records.add(manualSyncMockS3ds.addRecord(new StringInputStream("test3")));
+ records.add(manualSyncMockS3ds.addRecord(getStream("test1")));
+ records.add(manualSyncMockS3ds.addRecord(getStream("test2")));
+ records.add(manualSyncMockS3ds.addRecord(getStream("test3")));
- assertTrue(3 == mBean.getActiveSyncs());
+ assertTrue(3 == stats.getActiveSyncs());
} finally {
for (final DataRecord record : records) {
manualSyncMockS3ds.deleteRecord(record.getIdentifier());
@@ -181,125 +219,391 @@ public class S3DataStoreStatsTest {
((ManuallySyncingInMemoryBackend) manualSyncMockS3ds.getBackend()).clearInProgressWrites();
- assertTrue(0 == mBean.getActiveSyncs());
+ assertTrue(0 == stats.getActiveSyncs());
}
@Test
public void testIsFileSyncedMetricExists()
throws Exception {
- mBean = new S3DataStoreStats();
- mBean.nodeStore = mockNodeStore;
- mBean.s3ds = defaultS3ds;
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = defaultS3ds;
- assertFalse(mBean.isFileSynced(testNodePathName));
+ assertFalse(stats.isFileSynced(testNodePathName));
}
@Test
public void testIsFileSyncedNullFileReturnsFalse()
throws Exception {
- mBean = new S3DataStoreStats();
- mBean.nodeStore = mockNodeStore;
- mBean.s3ds = defaultS3ds;
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = defaultS3ds;
- assertFalse(mBean.isFileSynced(null));
+ assertFalse(stats.isFileSynced(null));
}
@Test
public void testIsFileSyncedEmptyStringReturnsFalse()
throws Exception {
- mBean = new S3DataStoreStats();
- mBean.nodeStore = mockNodeStore;
- mBean.s3ds = defaultS3ds;
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = defaultS3ds;
- assertFalse(mBean.isFileSynced(""));
+ assertFalse(stats.isFileSynced(""));
}
@Test
public void testIsFileSyncedInvalidFilenameReturnsFalse()
throws Exception {
- mBean = new S3DataStoreStats();
- mBean.nodeStore = mockNodeStore;
- mBean.s3ds = defaultS3ds;
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = defaultS3ds;
- assertFalse(mBean.isFileSynced("invalid"));
+ assertFalse(stats.isFileSynced("invalid"));
}
@Test
public void testIsFileSyncedFileNotAddedReturnsFalse()
throws Exception {
- mBean = new S3DataStoreStats();
- mBean.nodeStore = mockNodeStore;
- mBean.s3ds = autoSyncMockS3ds;
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = autoSyncMockS3ds;
- assertFalse(mBean.isFileSynced(testNodePathName));
+ assertFalse(stats.isFileSynced(testNodePathName));
}
@Test
public void testIsFileSyncedSyncIncompleteReturnsFalse()
throws Exception {
- mBean = new S3DataStoreStats();
- mBean.nodeStore = mockNodeStore;
- mBean.s3ds = manualSyncMockS3ds;
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = manualSyncMockS3ds;
- DataRecord record = null;
- FileInputStream stream = null;
- try {
- stream = new FileInputStream(testFile);
- record = manualSyncMockS3ds.addRecord(stream);
- assertFalse(mBean.isFileSynced(testNodePathName));
- } finally {
- IOUtils.closeQuietly(stream);
- if (null != record) {
- manualSyncMockS3ds.deleteRecord(record.getIdentifier());
- }
- }
+ assertSyncedFalse(stats, manualSyncMockS3ds, new FileInputStream(testFile));
}
@Test
public void testIsFileSyncedSyncCompleteReturnsTrue()
throws Exception {
- mBean = new S3DataStoreStats();
- mBean.nodeStore = mockNodeStore;
- mBean.s3ds = autoSyncMockS3ds;
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = autoSyncMockS3ds;
+
+ assertSyncedTrue(stats, autoSyncMockS3ds, new FileInputStream(testFile));
+ }
+
+ @Test
+ public void testIsFileSyncedFileDeletedReturnsFalse()
+ throws Exception {
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = autoSyncMockS3ds;
DataRecord record = null;
FileInputStream stream = null;
try {
stream = new FileInputStream(testFile);
record = autoSyncMockS3ds.addRecord(stream);
- assertTrue(mBean.isFileSynced(testNodePathName));
} finally {
IOUtils.closeQuietly(stream);
- if (null != record) {
- autoSyncMockS3ds.deleteRecord(record.getIdentifier());
+ delete(autoSyncMockS3ds, Lists.<DataRecord>newArrayList(record));
+ }
+ assertFalse(stats.isFileSynced(testNodePathName));
+ }
+
+ @Test
+ public void testIsFileSyncedDifferentPaths() throws Exception {
+ final String path1 = "path/to/node/1";
+ final String path2 = "path/to/node/2";
+ final String path3 = "shortpath";
+ final String path4 = "a/very/very/long/path/leads/to/node/4";
+ final List<String> paths = Lists.newArrayList(path1, path2, path3, path4);
+ final String leadingSlashPath = "/" + path1;
+
+ final List<String> blobContents = Lists.newArrayList("1", "2", "3", "4");
+ final List<Blob> blobs = Lists.newArrayList(
+ mock(Blob.class),
+ mock(Blob.class),
+ mock(Blob.class),
+ mock(Blob.class)
+ );
+ final List<String> blobIds = Lists.newArrayList(
+ getIdForInputStream(getStream(blobContents.get(0))),
+ getIdForInputStream(getStream(blobContents.get(1))),
+ getIdForInputStream(getStream(blobContents.get(2))),
+ getIdForInputStream(getStream(blobContents.get(3)))
+ );
+ when(blobs.get(0).getContentIdentity()).thenReturn(blobIds.get(0));
+ when(blobs.get(1).getContentIdentity()).thenReturn(blobIds.get(1));
+ when(blobs.get(2).getContentIdentity()).thenReturn(blobIds.get(2));
+ when(blobs.get(3).getContentIdentity()).thenReturn(blobIds.get(3));
+
+ final NodeStore nodeStore = new MemoryNodeStore();
+ final NodeBuilder rootBuilder = nodeStore.getRoot().builder();
+
+ final List<NodeBuilder> builders = Lists.newArrayList();
+ for (final String path : paths) {
+ NodeBuilder builder = rootBuilder;
+ for (final String nodeName : PathUtils.elements(path)) {
+ builder = builder.child(nodeName);
}
+ builders.add(builder);
+ }
+ builders.get(0).setProperty("blob1", blobs.get(0));
+ builders.get(1).setProperty("blob2", blobs.get(1));
+ builders.get(2).setProperty("blob3", blobs.get(2));
+ builders.get(3).setProperty("blob4", blobs.get(3));
+
+ nodeStore.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = autoSyncMockS3ds;
+
+ final List<DataRecord> records = Lists.newArrayList();
+ try {
+ for (final String s : blobContents) {
+ records.add(autoSyncMockS3ds.addRecord(getStream(s)));
+ }
+
+ for (final String path : Lists.newArrayList(path1, path2, path3, path4, leadingSlashPath)) {
+ assertTrue(stats.isFileSynced(path));
+ }
+
+ for (final String invalidPath : Lists.newArrayList(path1 + "/", "/" + path1 + "/", "/path//to/node///1")) {
+ try {
+ stats.isFileSynced(invalidPath);
+ assertFalse(false); // shouldn't get here on an invalid path
+ }
+ catch (AssertionError e) {
+ // expected
+ }
+ }
+ }
+ finally {
+ delete(autoSyncMockS3ds, records);
}
}
@Test
- public void testIsFileSyncedFileDeletedReturnsFalse()
- throws Exception {
- mBean = new S3DataStoreStats();
- mBean.nodeStore = mockNodeStore;
- mBean.s3ds = autoSyncMockS3ds;
+ public void testIsFileSyncedMultiplePropertiesReturnsTrue() throws Exception {
+ NodeStore nodeStore = initNodeStore(Optional.of(mockBlob),
+ Optional.<Blob>absent(),
+ Optional.of("abc"),
+ Optional.of(123),
+ Optional.<List<Blob>>absent());
+
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = autoSyncMockS3ds;
- DataRecord record = null;
- FileInputStream stream = null;
+ assertSyncedTrue(stats, autoSyncMockS3ds, new FileInputStream(testFile));
+ }
+
+ @Test
+ public void testIsFileSyncedMultipleBinaryPropertiesAllSyncedReturnsTrue() throws Exception {
+ Blob mockBlob2 = mock(Blob.class);
+ final String id2 = getIdForInputStream(getStream("testContents2"));
+ when(mockBlob2.getContentIdentity()).thenReturn(id2);
+ NodeStore nodeStore = initNodeStore(Optional.of(mockBlob),
+ Optional.of(mockBlob2),
+ Optional.<String>absent(),
+ Optional.<Integer>absent(),
+ Optional.<List<Blob>>absent());
+
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = autoSyncMockS3ds;
+
+ assertSyncedTrue(stats, autoSyncMockS3ds, new FileInputStream(testFile),
+ getStream("testContents2"));
+ }
+
+ @Test
+ public void testIsFileSyncedMultipleBinaryPropertiesNotAllSyncedReturnsFalse() throws Exception {
+ Blob mockBlob2 = mock(Blob.class);
+ final String id2 = getIdForInputStream(getStream("testContents2"));
+ when(mockBlob2.getContentIdentity()).thenReturn(id2);
+ NodeStore nodeStore = initNodeStore(Optional.of(mockBlob),
+ Optional.of(mockBlob2),
+ Optional.<String>absent(),
+ Optional.<Integer>absent(),
+ Optional.<List<Blob>>absent());
+
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = autoSyncMockS3ds;
+
+ assertSyncedFalse(stats, autoSyncMockS3ds, new FileInputStream(testFile));
+ }
+
+ @Test
+ public void testIsFileSyncedBinariesPropertySingleReturnsTrue() throws Exception {
+ List<Blob> blobPropList = Lists.newArrayList(mockBlob);
+ NodeStore nodeStore = initNodeStore(Optional.<Blob>absent(),
+ Optional.<Blob>absent(),
+ Optional.<String>absent(),
+ Optional.<Integer>absent(),
+ Optional.of(blobPropList));
+
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = autoSyncMockS3ds;
+
+ assertSyncedTrue(stats, autoSyncMockS3ds, new FileInputStream(testFile));
+ }
+
+ @Test
+ public void testIsFileSyncedBinariesPropertyMultipleReturnsTrue() throws Exception {
+ Blob mockBlob2 = mock(Blob.class);
+ final String id2 = getIdForInputStream(getStream("testContents2"));
+ when(mockBlob2.getContentIdentity()).thenReturn(id2);
+ List<Blob> blobPropList = Lists.newArrayList(mockBlob, mockBlob2);
+ NodeStore nodeStore = initNodeStore(Optional.<Blob>absent(),
+ Optional.<Blob>absent(),
+ Optional.<String>absent(),
+ Optional.<Integer>absent(),
+ Optional.of(blobPropList));
+
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = autoSyncMockS3ds;
+
+ assertSyncedTrue(stats, autoSyncMockS3ds, new FileInputStream(testFile),
+ getStream("testContents2"));
+ }
+
+ @Test
+ public void testIsFileSyncedBinariesPropertyNotAllSyncedReturnsFalse() throws Exception {
+ Blob mockBlob2 = mock(Blob.class);
+ final String id2 = getIdForInputStream(getStream("testContents2"));
+ when(mockBlob2.getContentIdentity()).thenReturn(id2);
+ List<Blob> blobPropList = Lists.newArrayList(mockBlob, mockBlob2);
+ NodeStore nodeStore = initNodeStore(Optional.<Blob>absent(),
+ Optional.<Blob>absent(),
+ Optional.<String>absent(),
+ Optional.<Integer>absent(),
+ Optional.of(blobPropList));
+
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = autoSyncMockS3ds;
+
+ assertSyncedFalse(stats, autoSyncMockS3ds, new FileInputStream(testFile));
+ }
+
+ @Test
+ public void testIsFileSyncedBinarySyncedAndBinariesNotSyncedReturnsFalse() throws Exception {
+ Blob mockBlob2 = mock(Blob.class);
+ final String id2 = getIdForInputStream(getStream("testContents2"));
+ when(mockBlob2.getContentIdentity()).thenReturn(id2);
+ Blob mockBlob3 = mock(Blob.class);
+ final String id3 = getIdForInputStream(getStream("testContents3"));
+ when(mockBlob2.getContentIdentity()).thenReturn(id3);
+ List<Blob> blobPropList = Lists.newArrayList(mockBlob2, mockBlob3);
+ NodeStore nodeStore = initNodeStore(Optional.of(mockBlob),
+ Optional.<Blob>absent(),
+ Optional.<String>absent(),
+ Optional.<Integer>absent(),
+ Optional.of(blobPropList));
+
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = autoSyncMockS3ds;
+
+ assertSyncedFalse(stats, autoSyncMockS3ds, new FileInputStream(testFile),
+ getStream("testContents2"));
+ }
+
+ @Test
+ public void testIsFileSyncedBinaryNotSyncedAndBinariesSyncedReturnsFalse() throws Exception {
+ Blob mockBlob2 = mock(Blob.class);
+ final String id2 = getIdForInputStream(getStream("testContents2"));
+ when(mockBlob2.getContentIdentity()).thenReturn(id2);
+ Blob mockBlob3 = mock(Blob.class);
+ final String id3 = getIdForInputStream(getStream("testContents3"));
+ when(mockBlob2.getContentIdentity()).thenReturn(id3);
+ List<Blob> blobPropList = Lists.newArrayList(mockBlob2, mockBlob3);
+ NodeStore nodeStore = initNodeStore(Optional.of(mockBlob),
+ Optional.<Blob>absent(),
+ Optional.<String>absent(),
+ Optional.<Integer>absent(),
+ Optional.of(blobPropList));
+
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = autoSyncMockS3ds;
+
+ assertSyncedFalse(stats, autoSyncMockS3ds, getStream("testContents2"),
+ getStream("testContents3"));
+ }
+
+ @Test
+ public void testIsFileSyncedBinaryAndBinariesSyncedReturnsTrue() throws Exception {
+ Blob mockBlob2 = mock(Blob.class);
+ final String id2 = getIdForInputStream(getStream("testContents2"));
+ when(mockBlob2.getContentIdentity()).thenReturn(id2);
+ Blob mockBlob3 = mock(Blob.class);
+ final String id3 = getIdForInputStream(getStream("testContents3"));
+ when(mockBlob2.getContentIdentity()).thenReturn(id3);
+ List<Blob> blobPropList = Lists.newArrayList(mockBlob2, mockBlob3);
+ NodeStore nodeStore = initNodeStore(Optional.of(mockBlob),
+ Optional.<Blob>absent(),
+ Optional.<String>absent(),
+ Optional.<Integer>absent(),
+ Optional.of(blobPropList));
+
+ stats = new S3DataStoreStats();
+ stats.nodeStore = nodeStore;
+ stats.s3ds = autoSyncMockS3ds;
+
+ assertSyncedFalse(stats, autoSyncMockS3ds, new FileInputStream(testFile),
+ getStream("testContents2"), getStream("testContents3"));
+ }
+
+ private static void delete(SharedS3DataStore s3ds, List<DataRecord> recs) throws DataStoreException {
+ for (DataRecord rec : recs) {
+ if (null != rec) {
+ s3ds.deleteRecord(rec.getIdentifier());
+ }
+ }
+ }
+
+ private static void assertSyncedFalse(S3DataStoreStats mBean, SharedS3DataStore s3ds,
+ InputStream... streams) throws DataStoreException {
+
+ List<DataRecord> recs = Lists.newArrayList();
try {
- stream = new FileInputStream(testFile);
- record = autoSyncMockS3ds.addRecord(stream);
- } finally {
- IOUtils.closeQuietly(stream);
- if (null != record) {
- autoSyncMockS3ds.deleteRecord(record.getIdentifier());
+ for (InputStream is : streams) {
+ recs.add(s3ds.addRecord(is));
+ IOUtils.closeQuietly(is);
}
+ assertFalse(mBean.isFileSynced(testNodePathName));
+ } finally {
+ delete(s3ds, recs);
}
+ }
- assertFalse(mBean.isFileSynced(testNodePathName));
+ private static void assertSyncedTrue(S3DataStoreStats mBean, SharedS3DataStore s3ds,
+ InputStream... streams) throws DataStoreException {
+
+ List<DataRecord> recs = Lists.newArrayList();
+ try {
+ for (InputStream is : streams) {
+ recs.add(s3ds.addRecord(is));
+ IOUtils.closeQuietly(is);
+ }
+ assertTrue(mBean.isFileSynced(testNodePathName));
+ } finally {
+ delete(s3ds, recs);
+ }
}
+ private static InputStream getStream(String str) {
+ return new ByteArrayInputStream(str.getBytes(Charsets.UTF_8));
+ }
// A mock S3DataStore that allows us to replace the default
// S3Backend with our own backend, for test purposes only.
@@ -405,7 +709,7 @@ public class S3DataStoreStatsTest {
}
- // A modified InMemoryBackend that, when writeAsync() is called, does not
+ // A modified TestMemoryBackend that, when writeAsync() is called, does not
// actually store the record but keeps track that it was intended to be
// stored, and allows the test to tell it when it expects the record
// to be "synced".
Copied: jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java (from r1760661, jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java?p2=jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java&p1=jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java&r1=1760661&r2=1760666&rev=1760666&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentS3DataStoreStatsTest.java Wed Sep 14 09:44:47 2016
@@ -106,7 +106,8 @@ public class DocumentS3DataStoreStatsTes
private void registerDocumentNodeStoreService(boolean customBlobStore) {
Map<String, Object> properties = newHashMap();
- properties.put("mongouri", MongoUtils.URL);
+ properties
+ .put("mongouri", String.format("mongodb://%s:%s", MongoUtils.HOST, MongoUtils.PORT));
properties.put("db", MongoUtils.DB);
properties.put(DocumentNodeStoreService.CUSTOM_BLOB_STORE, customBlobStore);
documentNodeStoreService =