You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2015/12/23 07:24:21 UTC
svn commit: r1721497 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/blob/
main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/
test/java/org/apache/jackrabbit/oak/plugins/blob/
test/java/org/apache/jackrabbi...
Author: chetanm
Date: Wed Dec 23 06:24:20 2015
New Revision: 1721497
URL: http://svn.apache.org/viewvc?rev=1721497&view=rev
Log:
OAK-3806 - Collect and expose statistics related to BlobStore operations
-- Add support to DataStoreBlobStore and expose the MBean for all DataStores
-- Add Stats based implementation with BlobStoreStats
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreStats.java (with props)
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreStatsTest.java (with props)
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreStatsTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreService.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreStats.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreStats.java?rev=1721497&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreStats.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreStats.java Wed Dec 23 06:24:20 2015
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.blob;
+
+import java.util.concurrent.TimeUnit;
+
+import javax.management.openmbean.CompositeData;
+
+import org.apache.jackrabbit.oak.spi.blob.stats.BlobStoreStatsMBean;
+import org.apache.jackrabbit.oak.spi.blob.stats.BlobStatsCollector;
+import org.apache.jackrabbit.oak.stats.HistogramStats;
+import org.apache.jackrabbit.oak.stats.MeterStats;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.apache.jackrabbit.stats.TimeSeriesStatsUtil;
+
+import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
+
+public class BlobStoreStats implements BlobStoreStatsMBean, BlobStatsCollector {
+ private static final String BLOB_UPLOADS = "BLOB_UPLOADS";
+ private static final String BLOB_DOWNLOADS = "BLOB_DOWNLOADS";
+
+ private final StatisticsProvider statisticsProvider;
+
+ private final HistogramStats uploadHisto;
+ private final MeterStats uploadSizeMeter;
+ private final MeterStats uploadTimeMeter;
+
+ private final HistogramStats downloadHisto;
+ private final MeterStats downloadSizeMeter;
+ private final MeterStats downloadTimeMeter;
+
+ public BlobStoreStats(StatisticsProvider sp) {
+ this.statisticsProvider = sp;
+
+ this.uploadHisto = sp.getHistogram(BLOB_UPLOADS);
+ //TODO Need to expose an API in StatisticsProvider to register for avg
+ //That would give us upload and download *rate*
+ this.uploadSizeMeter = sp.getMeter("BLOB_UPLOAD_SIZE");
+ this.uploadTimeMeter = sp.getMeter("BLOB_UPLOAD_TIME");
+
+ this.downloadHisto = sp.getHistogram(BLOB_DOWNLOADS);
+ this.downloadSizeMeter = sp.getMeter("BLOB_DOWNLOAD_SIZE");
+ this.downloadTimeMeter = sp.getMeter("BLOB_DOWNLOAD_TIME");
+ }
+
+ @Override
+ public void uploaded(long timeTaken, TimeUnit unit, long size) {
+ uploadHisto.update(size);
+
+ //Recording upload like this is not accurate. A more accurate way
+ //would be to mark as upload or download is progressing.
+ //That would however add quite a bit of overhead
+ //Approach below would record an upload/download at moment when
+ //it got completed. So acts like a rough approximation
+ uploadSizeMeter.mark(size);
+ uploadTimeMeter.mark(TimeUnit.NANOSECONDS.convert(timeTaken, unit));
+ }
+
+ @Override
+ public void downloaded(String blobId, long timeTaken, TimeUnit unit, long size) {
+ downloadHisto.update(size);
+ downloadSizeMeter.mark(size);
+ downloadTimeMeter.mark(TimeUnit.NANOSECONDS.convert(timeTaken, unit));
+ }
+
+ //~--------------------------------------< BlobStoreMBean >
+
+ @Override
+ public long getUploadTotalSize() {
+ return uploadSizeMeter.getCount();
+ }
+
+ @Override
+ public long getUploadCount() {
+ return uploadHisto.getCount();
+ }
+
+ @Override
+ public long getUploadTotalSeconds() {
+ return TimeUnit.NANOSECONDS.toSeconds(uploadTimeMeter.getCount());
+ }
+
+ @Override
+ public long getDownloadTotalSize() {
+ return downloadSizeMeter.getCount();
+ }
+
+ @Override
+ public long getDownloadCount() {
+ return downloadHisto.getCount();
+ }
+
+ @Override
+ public long getDownloadTotalSeconds() {
+ return TimeUnit.NANOSECONDS.toSeconds(downloadTimeMeter.getCount());
+ }
+
+ @Override
+ public String blobStoreInfoAsString() {
+ return String.format("Uploads - size = %s, count = %d%nDownloads - size = %s, count = %d",
+ humanReadableByteCount(getUploadTotalSize()),
+ getUploadCount(),
+ humanReadableByteCount(getDownloadTotalSize()),
+ getDownloadCount()
+ );
+ }
+
+ @Override
+ public CompositeData getUploadSizeHistory() {
+ return getTimeSeries(BLOB_UPLOADS);
+ }
+
+ @Override
+ public CompositeData getDownloadSizeHistory() {
+ return getTimeSeries(BLOB_DOWNLOADS);
+ }
+
+ private CompositeData getTimeSeries(String name){
+ return TimeSeriesStatsUtil.asCompositeData(statisticsProvider.getStats().getTimeSeries(name, true),
+ name);
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreStats.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreService.java?rev=1721497&r1=1721496&r2=1721497&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreService.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreService.java Wed Dec 23 06:24:20 2015
@@ -25,11 +25,18 @@ import java.util.Map;
import javax.jcr.RepositoryException;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Reference;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
+import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard;
+import org.apache.jackrabbit.oak.spi.blob.stats.BlobStoreStatsMBean;
+import org.apache.jackrabbit.oak.plugins.blob.BlobStoreStats;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
+import org.apache.jackrabbit.oak.spi.whiteboard.Registration;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.osgi.framework.Constants;
import org.osgi.framework.ServiceRegistration;
import org.osgi.service.component.ComponentContext;
@@ -37,7 +44,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.jackrabbit.oak.spi.blob.osgi.SplitBlobStoreService.PROP_SPLIT_BLOBSTORE;
+import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.registerMBean;
+@Component(componentAbstract = true)
public abstract class AbstractDataStoreService {
private static final String PROP_HOME = "repository.home";
@@ -47,8 +56,13 @@ public abstract class AbstractDataStoreS
private ServiceRegistration reg;
+ private Registration mbeanReg;
+
private Logger log = LoggerFactory.getLogger(getClass());
+ @Reference
+ private StatisticsProvider statisticsProvider;
+
private DataStore dataStore;
protected void activate(ComponentContext context, Map<String, Object> config) throws RepositoryException {
@@ -61,7 +75,8 @@ public abstract class AbstractDataStoreS
}
PropertiesUtil.populate(ds, config, false);
ds.init(homeDir);
- this.dataStore = new DataStoreBlobStore(ds, encodeLengthInId, cacheSizeInMB);
+ BlobStoreStats stats = new BlobStoreStats(getStatisticsProvider());
+ this.dataStore = new DataStoreBlobStore(ds, encodeLengthInId, cacheSizeInMB, stats);
PropertiesUtil.populate(dataStore, config, false);
Dictionary<String, Object> props = new Hashtable<String, Object>();
@@ -75,6 +90,12 @@ public abstract class AbstractDataStoreS
BlobStore.class.getName(),
GarbageCollectableBlobStore.class.getName()
}, dataStore , props);
+
+ mbeanReg = registerMBean(new OsgiWhiteboard(context.getBundleContext()),
+ BlobStoreStatsMBean.class,
+ stats,
+ BlobStoreStatsMBean.TYPE,
+ ds.getClass().getSimpleName());
}
protected void deactivate() throws DataStoreException {
@@ -82,11 +103,19 @@ public abstract class AbstractDataStoreS
reg.unregister();
}
+ if (mbeanReg != null){
+ mbeanReg.unregister();
+ }
+
dataStore.close();
}
protected abstract DataStore createDataStore(ComponentContext context, Map<String, Object> config);
+ protected StatisticsProvider getStatisticsProvider(){
+ return statisticsProvider;
+ }
+
protected String[] getDescription(){
return new String[] {"type=unknown"};
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1721497&r1=1721496&r2=1721497&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java Wed Dec 23 06:24:20 2015
@@ -34,6 +34,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -57,6 +58,8 @@ import org.apache.jackrabbit.oak.cache.C
import org.apache.jackrabbit.oak.commons.StringUtils;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.blob.stats.StatsCollectingStreams;
+import org.apache.jackrabbit.oak.spi.blob.stats.BlobStatsCollector;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,6 +75,8 @@ public class DataStoreBlobStore implemen
private final DataStore delegate;
+ private final BlobStatsCollector stats;
+
/**
* If set to true then the blob length information would be encoded as part of blobId
* and thus no extra call would be made to DataStore to determine the length
@@ -96,16 +101,18 @@ public class DataStoreBlobStore implemen
public DataStoreBlobStore(DataStore delegate) {
- this(delegate, true, DEFAULT_CACHE_SIZE);
+ this(delegate, true, DEFAULT_CACHE_SIZE, BlobStatsCollector.NOOP);
}
public DataStoreBlobStore(DataStore delegate, boolean encodeLengthInId) {
- this(delegate, encodeLengthInId, DEFAULT_CACHE_SIZE);
+ this(delegate, encodeLengthInId, DEFAULT_CACHE_SIZE, BlobStatsCollector.NOOP);
}
- public DataStoreBlobStore(DataStore delegate, boolean encodeLengthInId, int cacheSizeInMB) {
+ public DataStoreBlobStore(DataStore delegate, boolean encodeLengthInId, int cacheSizeInMB,
+ BlobStatsCollector stats) {
this.delegate = delegate;
this.encodeLengthInId = encodeLengthInId;
+ this.stats = stats;
this.cache = CacheLIRS.<String, byte[]>newBuilder()
.module("DataStoreBlobStore")
@@ -188,10 +195,12 @@ public class DataStoreBlobStore implemen
public String writeBlob(InputStream stream) throws IOException {
boolean threw = true;
try {
+ long start = System.nanoTime();
checkNotNull(stream);
DataRecord dr = writeStream(stream);
String id = getBlobId(dr);
threw = false;
+ stats.uploaded(System.nanoTime() - start, TimeUnit.NANOSECONDS, dr.getLength());
return id;
} catch (DataStoreException e) {
throw new IOException(e);
@@ -477,7 +486,7 @@ public class DataStoreBlobStore implemen
if (!(in instanceof BufferedInputStream)){
in = new BufferedInputStream(in);
}
- return in;
+ return StatsCollectingStreams.wrap(stats, blobId, in);
} catch (DataStoreException e) {
throw new IOException(e);
}
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreStatsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreStatsTest.java?rev=1721497&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreStatsTest.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreStatsTest.java Wed Dec 23 06:24:20 2015
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.blob;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.junit.After;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class BlobStoreStatsTest {
+ private ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+ private StatisticsProvider statsProvider = new DefaultStatisticsProvider(executor);
+ private BlobStoreStats stats = new BlobStoreStats(statsProvider);
+
+ @After
+ public void shutDown(){
+ new ExecutorCloser(executor).close();
+ }
+
+ @Test
+ public void upload() throws Exception{
+ stats.uploaded(103, TimeUnit.SECONDS, 1079);
+ assertEquals(103, stats.getUploadTotalSeconds());
+ assertEquals(1079, stats.getUploadTotalSize());
+ assertEquals(1, stats.getUploadCount());
+
+ stats.uploaded(53, TimeUnit.SECONDS, 47);
+ assertEquals(103 + 53, stats.getUploadTotalSeconds());
+ assertEquals(1079 + 47, stats.getUploadTotalSize());
+ assertEquals(2, stats.getUploadCount());
+ }
+
+ @Test
+ public void download() throws Exception{
+ stats.downloaded("foo", 103, TimeUnit.SECONDS, 1079);
+ assertEquals(103, stats.getDownloadTotalSeconds());
+ assertEquals(1079, stats.getDownloadTotalSize());
+ assertEquals(1, stats.getDownloadCount());
+
+ stats.downloaded("foo", 53, TimeUnit.SECONDS, 47);
+ assertEquals(103 + 53, stats.getDownloadTotalSeconds());
+ assertEquals(1079 + 47, stats.getDownloadTotalSize());
+ assertEquals(2, stats.getDownloadCount());
+ }
+}
\ No newline at end of file
Propchange: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreStatsTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java?rev=1721497&r1=1721496&r2=1721497&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java Wed Dec 23 06:24:20 2015
@@ -113,7 +113,7 @@ public class DataStoreBlobStoreTest exte
assertEquals(dr, ds.getRecordIfStored(dr.getIdentifier()));
assertEquals(dr, ds.getRecord(dr.getIdentifier()));
- assertTrue(ds.getInputStream(dr.getIdentifier().toString()) instanceof BufferedInputStream);
+// assertTrue(ds.getInputStream(dr.getIdentifier().toString()) instanceof BufferedInputStream);
assertEquals(actualSize, ds.getBlobLength(dr.getIdentifier().toString()));
assertEquals(testDI.toString(), BlobId.of(ds.writeBlob(new ByteArrayInputStream(data))).blobId);
}
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreStatsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreStatsTest.java?rev=1721497&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreStatsTest.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreStatsTest.java Wed Dec 23 06:24:20 2015
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.blob.datastore;
+
+import org.apache.jackrabbit.core.data.FileDataStore;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.blob.StatsCollectorTest;
+import org.apache.jackrabbit.oak.spi.blob.stats.BlobStatsCollector;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+
+public class DataStoreStatsTest extends StatsCollectorTest {
+ @Rule
+ public final TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Override
+ protected BlobStore createBlobStore(BlobStatsCollector collector) {
+ FileDataStore fds = DataStoreUtils.createFDS(temporaryFolder.getRoot(), 0);
+ return new DataStoreBlobStore(fds, true, 1, collector);
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreStatsTest.java
------------------------------------------------------------------------------
svn:eol-style = native