You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mi...@apache.org on 2022/09/27 14:15:23 UTC
[jackrabbit-oak] branch trunk updated: OAK-9928 Add benchmark for access of binary values in cloud DataStore
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new f9e8d18235 OAK-9928 Add benchmark for access of binary values in cloud DataStore
new e4cbe4c518 Merge pull request #698 from jelmini/OAK-9928_binaries_benchmark
f9e8d18235 is described below
commit f9e8d1823571408d03b010fb5354961721ce6343
Author: Carlo Jelmini <je...@adobe.com>
AuthorDate: Tue Sep 6 15:09:28 2022 +0200
OAK-9928 Add benchmark for access of binary values in cloud DataStore
---
oak-benchmarks/run_azure_datastore.sh | 78 +++++++++++++
.../jackrabbit/oak/benchmark/BenchmarkOptions.java | 11 ++
.../jackrabbit/oak/benchmark/BenchmarkRunner.java | 16 ++-
.../oak/benchmark/ReadBinaryPropertiesTest.java | 127 +++++++++++++++++++++
.../oak/scalability/ScalabilityRunner.java | 6 +-
.../oak/fixture/CompositeStoreFixture.java | 5 +-
.../jackrabbit/oak/fixture/DataStoreUtils.java | 29 +++--
.../apache/jackrabbit/oak/fixture/OakFixture.java | 46 +++++---
.../oak/fixture/OakRepositoryFixture.java | 29 ++---
.../jackrabbit/oak/fixture/SegmentTarFixture.java | 13 ++-
10 files changed, 308 insertions(+), 52 deletions(-)
diff --git a/oak-benchmarks/run_azure_datastore.sh b/oak-benchmarks/run_azure_datastore.sh
new file mode 100755
index 0000000000..4ddbcdbfb5
--- /dev/null
+++ b/oak-benchmarks/run_azure_datastore.sh
@@ -0,0 +1,78 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+TITLE=AzureDatastoreTest
+BENCH="ReadBinaryPropertiesTest"
+ADMIN="false" # true"
+RUNTIME=360
+RANDOM_USER="true"
+FIXS="Oak-Segment-Azure"
+PROFILE=false
+#DEBUG=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005
+DEBUG=""
+
+AZURITE="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1"
+# Set DataStore connection string using DS_AZURE_CONNECTION_STRING environment variable. If not set, defaults to Azurite default test credentials
+DATASTORE_AZURE_CONNECTION=${DS_AZURE_CONNECTION_STRING:-$AZURITE}
+# Set SegmentStore connection string using AZURE_CONNECTION_STRING environment variable. If not set, defaults to Azurite default test credentials
+AZURE_CONNECTION=${AZURE_CONNECTION_STRING:-$AZURITE}
+AZURE_CONTAINER="segment"
+AZURE_ROOT_PATH="/aem"
+DATASTORE_CLASS="org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzureDataStore"
+BINARIES_THRESHOLD="512"
+TREE_MAX_NODES=300
+TREE_BINARY_SIZE=100000
+
+LOG=$TITLE"_$(date +'%Y%m%d_%H%M%S').csv"
+echo "Benchmarks: $BENCH" > $LOG
+echo "Fixtures: $FIXS" >> $LOG
+echo "Admin User: $ADMIN" >> $LOG
+echo "Runtime: $RUNTIME" >> $LOG
+echo "Random User: $RANDOM_USER" >> $LOG
+echo "Profiling: $PROFILE" >> $LOG
+echo "Tree - Max nodes: $TREE_MAX_NODES" >> $LOG
+echo "Tree - binary value size: $TREE_BINARY_SIZE" >> $LOG
+echo "--------------------------------------" >> $LOG
+
+rm -rf target/oak-benchmarks-*-tests.jar # remove unused but conflicting jar, if present
+
+# Generate Datastore config file. Escape `=` in connection string
+echo "azureConnectionString=\"$DATASTORE_AZURE_CONNECTION\"" | sed -e 's/=/\\=/g' -e 's/\\=/=/1' > target/datastore.properties
+# Set DataStore#minRecordLength to same value as BINARIES_THRESHOLD to force writing binary records to the BlobStore instead of keeping them in memory
+echo "minRecordLength=\"$BINARIES_THRESHOLD\"" >> target/datastore.properties
+# Disable local caching of binary data
+echo "cacheSize=\"0\"" >> target/datastore.properties
+
+for bm in $BENCH
+ do
+ for user in $ADMIN
+ do
+ # we start new VMs for each fixture to minimize memory impacts between them
+ for fix in $FIXS
+ do
+ echo "Executing benchmarks as admin: $user on $fix" | tee -a $LOG
+ echo "-----------------------------------------------------------" | tee -a $LOG
+ rm -rf target/$FIXS-* # Remove dangling local repository caches, if any
+ cmd="java $DEBUG -Xmx2048m -Dprofile=$PROFILE -Druntime=$RUNTIME -DskipWarmup=true -DmaxNodes=$TREE_MAX_NODES -DbinaryValueSize=$TREE_BINARY_SIZE -DdataStore=$DATASTORE_CLASS -Dds.config=target/datastore.properties -jar target/oak-benchmarks-*.jar benchmark --csvFile $LOG --runAsAdmin $user --report false --randomUser $RANDOM_USER --binariesInlineThreshold $BINARIES_THRESHOLD --azure $AZURE_CONNECTION --azureContainerName $AZURE_CONTAINER --azureRootPath $AZURE_ROOT_PATH $bm $fix"
+ echo $cmd
+ $cmd
+ done
+ done
+done
+echo "-----------------------------------------"
+echo "Benchmark completed. see $LOG for details:"
+cat $LOG
diff --git a/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkOptions.java b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkOptions.java
index 8044692761..0f229b15f4 100644
--- a/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkOptions.java
+++ b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkOptions.java
@@ -24,6 +24,7 @@ import joptsimple.OptionParser;
import joptsimple.OptionSpec;
import org.apache.jackrabbit.oak.benchmark.authorization.AceCreationTest;
import org.apache.jackrabbit.oak.security.authorization.composite.CompositeAuthorizationConfiguration;
+import org.apache.jackrabbit.oak.segment.Segment;
import org.apache.jackrabbit.oak.spi.xml.ImportBehavior;
import static java.util.Arrays.asList;
@@ -48,6 +49,7 @@ public class BenchmarkOptions {
private final OptionSpec<String> azureContainerName;
private final OptionSpec<String> azureRootPath;
private final OptionSpec<Boolean> mmap;
+ private final OptionSpec<Integer> binariesInlineThreshold;
private final OptionSpec<Integer> cache;
private final OptionSpec<Integer> fdsCache;
private final OptionSpec<File> wikipedia;
@@ -178,6 +180,10 @@ public class BenchmarkOptions {
return mmap;
}
+ public OptionSpec<Integer> getBinariesInlineThreshold() {
+ return binariesInlineThreshold;
+ }
+
public OptionSpec<Integer> getCache() {
return cache;
}
@@ -402,6 +408,11 @@ public class BenchmarkOptions {
mmap = parser.accepts("mmap", "TarMK memory mapping")
.withOptionalArg().ofType(Boolean.class)
.defaultsTo("64".equals(System.getProperty("sun.arch.data.model")));
+
+ binariesInlineThreshold = parser.accepts("binariesInlineThreshold", "TarMK binaries inline threshold")
+ .withOptionalArg().ofType(Integer.class)
+ .defaultsTo(Segment.MEDIUM_LIMIT);
+
cache = parser.accepts("cache", "cache size (MB)")
.withRequiredArg().ofType(Integer.class).defaultsTo(100);
fdsCache = parser.accepts("blobCache", "cache size (MB)")
diff --git a/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
index d30a28c67a..aaac39bee1 100644
--- a/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
+++ b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
@@ -114,11 +114,13 @@ public class BenchmarkRunner {
benchmarkOptions.getDropDBAfterTest().value(options),
cacheSize * MB, benchmarkOptions.isThrottlingEnabled().value(options)),
OakRepositoryFixture.getSegmentTar(benchmarkOptions.getBase().value(options), 256, cacheSize,
- benchmarkOptions.getMmap().value(options)),
+ benchmarkOptions.getMmap().value(options), benchmarkOptions.getBinariesInlineThreshold().value(options)),
OakRepositoryFixture.getSegmentTarWithDataStore(benchmarkOptions.getBase().value(options), 256, cacheSize,
- benchmarkOptions.getMmap().value(options), benchmarkOptions.getFdsCache().value(options)),
+ benchmarkOptions.getMmap().value(options), benchmarkOptions.getBinariesInlineThreshold().value(options),
+ benchmarkOptions.getFdsCache().value(options)),
OakRepositoryFixture.getSegmentTarWithColdStandby(benchmarkOptions.getBase().value(options), 256, cacheSize,
- benchmarkOptions.getMmap().value(options), benchmarkOptions.getColdUseDataStore().value(options),
+ benchmarkOptions.getMmap().value(options), benchmarkOptions.getBinariesInlineThreshold().value(options),
+ benchmarkOptions.getColdUseDataStore().value(options),
benchmarkOptions.getFdsCache().value(options),
benchmarkOptions.getColdSyncInterval().value(options),
benchmarkOptions.getColdShareDataStore().value(options), benchmarkOptions.getColdSecure().value(options),
@@ -127,7 +129,8 @@ public class BenchmarkRunner {
benchmarkOptions.getAzureConnectionString().value(options),
benchmarkOptions.getAzureContainerName().value(options),
benchmarkOptions.getAzureRootPath().value(options),
- 256, cacheSize, true, benchmarkOptions.getFdsCache().value(options)),
+ 256, cacheSize, benchmarkOptions.getBinariesInlineThreshold().value(options),
+ true, benchmarkOptions.getFdsCache().value(options)),
OakRepositoryFixture.getRDB(benchmarkOptions.getRdbjdbcuri().value(options),
benchmarkOptions.getRdbjdbcuser().value(options),
benchmarkOptions.getRdbjdbcpasswd().value(options), benchmarkOptions.getRdbjdbctableprefix().value(options),
@@ -138,7 +141,7 @@ public class BenchmarkRunner {
benchmarkOptions.getDropDBAfterTest().value(options), cacheSize * MB, benchmarkOptions.getBase().value(options),
benchmarkOptions.getFdsCache().value(options), benchmarkOptions.getVgcMaxAge().value(options)),
OakRepositoryFixture.getCompositeStore(benchmarkOptions.getBase().value(options), 256, cacheSize,
- benchmarkOptions.getMmap().value(options)),
+ benchmarkOptions.getMmap().value(options), benchmarkOptions.getBinariesInlineThreshold().value(options)),
OakRepositoryFixture.getCompositeMemoryStore(),
OakRepositoryFixture.getCompositeMongoStore(uri, cacheSize * MB,
benchmarkOptions.getDropDBAfterTest().value(options), benchmarkOptions.isThrottlingEnabled().value(options))
@@ -456,7 +459,8 @@ public class BenchmarkRunner {
new IsNodeTypeTest(benchmarkOptions.getRunAsAdmin().value(options)),
new SetPropertyTransientTest(),
new GetURITest(),
- new ISO8601FormatterTest()
+ new ISO8601FormatterTest(),
+ new ReadBinaryPropertiesTest()
)
);
diff --git a/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/ReadBinaryPropertiesTest.java b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/ReadBinaryPropertiesTest.java
new file mode 100644
index 0000000000..6fe600da71
--- /dev/null
+++ b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/ReadBinaryPropertiesTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+import java.io.ByteArrayInputStream;
+import java.util.UUID;
+import javax.jcr.Binary;
+import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.Property;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.commons.lang3.time.StopWatch;
+
+public class ReadBinaryPropertiesTest extends AbstractTest<Void> {
+
+ private static final int BINARY_SIZE = Integer.getInteger("binaryValueSize", 32000);
+ private static final int MAX_NODES = Integer.getInteger("maxNodes", 500);
+ private static final int MAX_CHILDREN_PER_NODE = 10;
+
+ @Override
+ protected void beforeTest() throws Exception {
+ // To prevent Oak caching to affect the performance stats, we create a new tree of nodes with binary properties for each test execution pass.
+ Session session = loginWriter();
+ Node rootNode = session.getRootNode();
+ System.out.printf("Creating tree with %d nodes. Binary size: %d%n", MAX_NODES, BINARY_SIZE);
+ int maxDepth = computeDepth(MAX_NODES);
+ StopWatch watch = StopWatch.createStarted();
+ int nodeCount = createTree(rootNode, maxDepth, 0);
+ watch.stop();
+ System.out.printf("%d nodes created in %dms%n", nodeCount, watch.getTime());
+ }
+
+ /**
+ * Compute depth necessary for the tree to contain {@code maxNodes}, given {@link #MAX_CHILDREN_PER_NODE} children for each node.
+ */
+ private static int computeDepth(int maxNodes) {
+ double nodes = 0;
+ int depth = 1;
+ while (true) {
+ nodes += Math.pow(MAX_CHILDREN_PER_NODE, depth);
+ if (maxNodes / nodes <= 1) {
+ return depth;
+ }
+ depth++;
+ }
+ }
+
+ private static int createTree(Node rootNode, int depth, int nodeCount) throws Exception {
+ Session session = rootNode.getSession();
+ for (int i = 0; i < MAX_CHILDREN_PER_NODE && nodeCount < MAX_NODES; i++) {
+ Node child = rootNode.addNode(UUID.randomUUID().toString(), "oak:Unstructured");
+ byte[] b = RandomUtils.nextBytes(BINARY_SIZE);
+ Binary binary = session.getValueFactory().createBinary(new ByteArrayInputStream(b));
+ child.setProperty("bin", binary);
+ nodeCount++;
+ if (depth > 0 && nodeCount < MAX_NODES) {
+ nodeCount = createTree(child, depth - 1, nodeCount);
+ }
+ }
+ session.save();
+ return nodeCount;
+ }
+
+ @Override
+ protected void runTest() throws Exception {
+ Session session = loginWriter();
+ Node rootNode = session.getRootNode();
+ traverseTree(rootNode);
+ System.out.println("Traversal done");
+ }
+
+ private static void traverseTree(Node rootNode) throws Exception {
+ NodeIterator nodes = rootNode.getNodes();
+ while (nodes.hasNext()) {
+ Node child = nodes.nextNode();
+ if (!child.hasProperty("bin")) {
+ continue; // ignore nodes auto-generated by Oak
+ }
+ // read binary property
+ Property property = child.getProperty("bin");
+ byte[] bytes = IOUtils.toByteArray(property.getBinary().getStream());
+ if (bytes.length != BINARY_SIZE) {
+ System.out.println("Invalid binary property in node " + child.getPath());
+ }
+ if (child.hasNodes()) {
+ traverseTree(child);
+ }
+ }
+ }
+
+ @Override
+ protected void afterTest() throws Exception {
+ Session session = loginWriter();
+ Node rootNode = session.getRootNode();
+ deleteTree(session, rootNode);
+ }
+
+ private static void deleteTree(Session session, Node rootNode) throws RepositoryException {
+ NodeIterator nodes = rootNode.getNodes();
+ while (nodes.hasNext()) {
+ Node child = nodes.nextNode();
+ if (child.hasProperty("bin")) {
+ child.remove();
+ }
+ }
+ session.save();
+ }
+}
diff --git a/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/ScalabilityRunner.java b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/ScalabilityRunner.java
index d85e863c02..d38f251106 100644
--- a/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/ScalabilityRunner.java
+++ b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/ScalabilityRunner.java
@@ -41,6 +41,7 @@ import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
import org.apache.jackrabbit.oak.scalability.benchmarks.segment.standby.StandbyBulkTransferBenchmark;
import org.apache.jackrabbit.oak.scalability.suites.ScalabilityStandbySuite;
+import org.apache.jackrabbit.oak.segment.Segment;
/**
* Main class for running scalability/longevity tests.
@@ -94,13 +95,14 @@ public class ScalabilityRunner {
scalabilityOptions.isThrottlingEnabled().value(options)),
OakRepositoryFixture.getSegmentTar(
scalabilityOptions.getBase().value(options), 256, cacheSize,
- scalabilityOptions.getMmap().value(options)),
+ scalabilityOptions.getMmap().value(options), Segment.MEDIUM_LIMIT),
OakRepositoryFixture.getSegmentTarWithDataStore(scalabilityOptions.getBase().value(options),
256, cacheSize,
- scalabilityOptions.getMmap().value(options),
+ scalabilityOptions.getMmap().value(options), Segment.MEDIUM_LIMIT,
scalabilityOptions.getFdsCache().value(options)),
OakRepositoryFixture.getSegmentTarWithColdStandby(scalabilityOptions.getBase().value(options), 256, cacheSize,
scalabilityOptions.getMmap().value(options),
+ Segment.MEDIUM_LIMIT,
scalabilityOptions.getColdUseDataStore().value(options),
scalabilityOptions.getFdsCache().value(options),
scalabilityOptions.getColdSyncInterval().value(options),
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/CompositeStoreFixture.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/CompositeStoreFixture.java
index 8af500ea96..4cfc278b98 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/CompositeStoreFixture.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/CompositeStoreFixture.java
@@ -79,7 +79,7 @@ abstract class CompositeStoreFixture extends OakFixture {
}
static OakFixture newCompositeSegmentFixture(String name, File base, int maxFileSizeMB, int cacheSizeMB,
- boolean memoryMapping) {
+ boolean memoryMapping, int binariesInlineThreshold) {
return new CompositeStoreFixture(name) {
private FileStore fileStore;
@@ -90,7 +90,8 @@ abstract class CompositeStoreFixture extends OakFixture {
FileStoreBuilder fsBuilder = fileStoreBuilder(new File(base, unique))
.withMaxFileSize(maxFileSizeMB)
.withSegmentCacheSize(cacheSizeMB)
- .withMemoryMapping(memoryMapping);
+ .withMemoryMapping(memoryMapping)
+ .withBinariesInlineThreshold(binariesInlineThreshold);
fileStore = fsBuilder.build();
return SegmentNodeStoreBuilders.builder(fileStore).build();
}
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
index 018f8ba4e5..2afba2a5a2 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/DataStoreUtils.java
@@ -103,7 +103,8 @@ public class DataStoreUtils {
if (!Strings.isNullOrEmpty(bucket)) {
deleteBucket(bucket, config, new Date());
}
- } else if (config.containsKey(AzureConstants.AZURE_BLOB_CONTAINER_NAME)) {
+ } else if (config.containsKey(AzureConstants.AZURE_BLOB_CONTAINER_NAME)
+ || config.containsKey(AzureConstants.AZURE_CONNECTION_STRING)) {
deleteAzureContainer(config, bucket);
}
}
@@ -145,20 +146,26 @@ public class DataStoreUtils {
}
public static void deleteAzureContainer(Map<String, ?> config, String containerName) throws Exception {
- String accountName = (String)config.get(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME);
- String accountKey = (String)config.get(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY);
- if (Strings.isNullOrEmpty(containerName) ||
- Strings.isNullOrEmpty(accountName) ||
- Strings.isNullOrEmpty(accountKey)) {
+ if (Strings.isNullOrEmpty(containerName)) {
return;
}
- log.info("deleting container [" + containerName + "]");
- CloudBlobContainer container = org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils
- .getBlobContainer(org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils.getConnectionString(accountName, accountKey), containerName);
+ String connectionString = (String) config.get(AzureConstants.AZURE_CONNECTION_STRING);
+ if (Strings.isNullOrEmpty(connectionString)) {
+ String accountName = (String) config.get(AzureConstants.AZURE_STORAGE_ACCOUNT_NAME);
+ String accountKey = (String) config.get(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY);
+ if (Strings.isNullOrEmpty(accountName) ||
+ Strings.isNullOrEmpty(accountKey)) {
+ return;
+ }
+ connectionString = org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils.getConnectionString(accountName, accountKey);
+ }
+ log.info("deleting container [{}]", containerName);
+ CloudBlobContainer container =
+ org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.Utils.getBlobContainer(connectionString, containerName);
if (container.deleteIfExists()) {
- log.info("container [ " + containerName + "] deleted");
+ log.info("container [{}] deleted", containerName);
} else {
- log.info("container [" + containerName + "] doesn't exists");
+ log.info("container [{}] doesn't exists", containerName);
}
}
}
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java
index a820e5c5e8..1dd1a0fbf9 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java
@@ -45,6 +45,7 @@ import org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore;
import org.apache.jackrabbit.oak.plugins.document.rdb.RDBOptions;
import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.segment.Segment;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.filter.PathFilter;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
@@ -324,62 +325,73 @@ public abstract class OakFixture {
}
public static OakFixture getSegmentTar(final String name, final File base, final int maxFileSizeMB,
- final int cacheSizeMB, final boolean memoryMapping, final boolean useBlobStore, final int dsCacheInMB,
- final boolean withColdStandby, final int syncInterval, final boolean shareBlobStore, final boolean secure,
- final boolean oneShotRun) {
+ final int cacheSizeMB, final boolean memoryMapping, final int binariesInlineThreshold, final boolean useBlobStore,
+ final int dsCacheInMB, final boolean withColdStandby, final int syncInterval, final boolean shareBlobStore,
+ final boolean secure, final boolean oneShotRun) {
SegmentTarFixtureBuilder builder = SegmentTarFixtureBuilder.segmentTarFixtureBuilder(name, base);
builder.withMaxFileSize(maxFileSizeMB).withSegmentCacheSize(cacheSizeMB).withMemoryMapping(memoryMapping)
+ .withBinariesInlineThreshold(binariesInlineThreshold)
.withBlobStore(useBlobStore).withDSCacheSize(dsCacheInMB);
return new SegmentTarFixture(builder, withColdStandby, syncInterval, shareBlobStore, secure, oneShotRun);
}
public static OakFixture getVanillaSegmentTar(final File base, final int maxFileSizeMB,
- final int cacheSizeMB, final boolean memoryMapping) {
+ final int cacheSizeMB, final boolean memoryMapping, final int binariesInlineThreshold) {
- return getSegmentTar(OakFixture.OAK_SEGMENT_TAR, base, maxFileSizeMB, cacheSizeMB, memoryMapping, false, 0,
+ return getSegmentTar(OakFixture.OAK_SEGMENT_TAR, base, maxFileSizeMB, cacheSizeMB, memoryMapping, binariesInlineThreshold, false, 0,
false, -1, false, false, false);
}
+ public static OakFixture getVanillaSegmentTar(final File base, final int maxFileSizeMB,
+ final int cacheSizeMB, final boolean memoryMapping) {
+
+ return getVanillaSegmentTar(base, maxFileSizeMB, cacheSizeMB, memoryMapping, Segment.MEDIUM_LIMIT);
+ }
+
public static OakFixture getSegmentTarWithDataStore(final File base,
- final int maxFileSizeMB, final int cacheSizeMB, final boolean memoryMapping, final int dsCacheInMB) {
+ final int maxFileSizeMB, final int cacheSizeMB, final boolean memoryMapping, final int binariesInlineThreshold,
+ final int dsCacheInMB) {
- return getSegmentTar(OakFixture.OAK_SEGMENT_TAR_DS, base, maxFileSizeMB, cacheSizeMB, memoryMapping, true, dsCacheInMB,
- false, -1, false, false, false);
+ return getSegmentTar(OakFixture.OAK_SEGMENT_TAR_DS, base, maxFileSizeMB, cacheSizeMB, memoryMapping, binariesInlineThreshold, true,
+ dsCacheInMB, false, -1, false, false, false);
}
public static OakFixture getSegmentTarWithColdStandby(final File base, final int maxFileSizeMB,
- final int cacheSizeMB, final boolean memoryMapping, final boolean useBlobStore, final int dsCacheInMB,
- final int syncInterval, final boolean shareBlobStore, final boolean secure, final boolean oneShotRun) {
+ final int cacheSizeMB, final boolean memoryMapping, final int binariesInlineThreshold, final boolean useBlobStore,
+ final int dsCacheInMB, final int syncInterval, final boolean shareBlobStore, final boolean secure, final boolean oneShotRun) {
- return getSegmentTar(OakFixture.OAK_SEGMENT_TAR_COLD, base, maxFileSizeMB, cacheSizeMB, memoryMapping, useBlobStore,
- dsCacheInMB, true, syncInterval, shareBlobStore, secure, oneShotRun);
+ return getSegmentTar(OakFixture.OAK_SEGMENT_TAR_COLD, base, maxFileSizeMB, cacheSizeMB, memoryMapping, binariesInlineThreshold,
+ useBlobStore, dsCacheInMB, true, syncInterval, shareBlobStore, secure, oneShotRun);
}
public static OakFixture getSegmentTarWithAwsSegmentStore(final File base, final String awsBucketName,
final String awsRootPath, final String awsJournalTableName, final String awsLockTableName,
- final int maxFileSizeMB, final int cacheSizeMB, final boolean useBlobStore, final int dsCacheInMB) {
+ final int maxFileSizeMB, final int cacheSizeMB, int binariesInlineThreshold, final boolean useBlobStore, final int dsCacheInMB) {
return SegmentTarFixtureBuilder.segmentTarFixtureBuilder(OakFixture.OAK_SEGMENT_AWS, base)
.withAws(awsBucketName, awsRootPath, awsJournalTableName, awsLockTableName)
- .withMaxFileSize(maxFileSizeMB).withSegmentCacheSize(cacheSizeMB).withBlobStore(useBlobStore)
+ .withMaxFileSize(maxFileSizeMB).withSegmentCacheSize(cacheSizeMB)
+ .withBinariesInlineThreshold(binariesInlineThreshold).withBlobStore(useBlobStore)
.withDSCacheSize(dsCacheInMB).build();
}
public static OakFixture getSegmentTarWithAzureSegmentStore(final File base, final String azureConnectionString, final String azureContainerName, final String azureRootPath,
- final int maxFileSizeMB, final int cacheSizeMB, final boolean useBlobStore, final int dsCacheInMB) {
+ final int maxFileSizeMB, final int cacheSizeMB, int binariesInlineThreshold, final boolean useBlobStore, final int dsCacheInMB) {
return SegmentTarFixtureBuilder
.segmentTarFixtureBuilder(OakFixture.OAK_SEGMENT_AZURE, base)
.withAzure(azureConnectionString, azureContainerName, azureRootPath)
.withMaxFileSize(maxFileSizeMB)
.withSegmentCacheSize(cacheSizeMB)
+ .withBinariesInlineThreshold(binariesInlineThreshold)
.withBlobStore(useBlobStore)
.withDSCacheSize(dsCacheInMB).build();
}
public static OakFixture getCompositeStore(final String name, final File base,
- final int maxFileSizeMB, final int cacheSizeMB, final boolean memoryMapping) {
- return newCompositeSegmentFixture(name, base, maxFileSizeMB, cacheSizeMB, memoryMapping);
+ final int maxFileSizeMB, final int cacheSizeMB, final boolean memoryMapping,
+ int binariesInlineThreshold) {
+ return newCompositeSegmentFixture(name, base, maxFileSizeMB, cacheSizeMB, memoryMapping, binariesInlineThreshold);
}
public static OakFixture getCompositeMemoryStore(final String name) {
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
index bf4eeccb1a..0f9de72217 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
@@ -92,43 +92,46 @@ public class OakRepositoryFixture implements RepositoryFixture {
}
public static RepositoryFixture getSegmentTar(File base, int maxFileSizeMB, int cacheSizeMB,
- boolean memoryMapping) {
+ boolean memoryMapping, final int binariesInlineThreshold) {
return new OakRepositoryFixture(
- OakFixture.getVanillaSegmentTar(base, maxFileSizeMB, cacheSizeMB, memoryMapping));
+ OakFixture.getVanillaSegmentTar(base, maxFileSizeMB, cacheSizeMB, memoryMapping, binariesInlineThreshold));
}
public static RepositoryFixture getSegmentTarWithDataStore(File base, int maxFileSizeMB, int cacheSizeMB,
- boolean memoryMapping, int dsCacheInMB) {
+ boolean memoryMapping, final int binariesInlineThreshold, int dsCacheInMB) {
return new OakRepositoryFixture(
- OakFixture.getSegmentTarWithDataStore(base, maxFileSizeMB, cacheSizeMB, memoryMapping, dsCacheInMB));
+ OakFixture.getSegmentTarWithDataStore(base, maxFileSizeMB, cacheSizeMB, memoryMapping, binariesInlineThreshold,
+ dsCacheInMB));
}
public static RepositoryFixture getSegmentTarWithColdStandby(File base, int maxFileSizeMB, int cacheSizeMB,
- boolean memoryMapping, boolean useBlobStore, int dsCacheInMB, int syncInterval, boolean shareBlobStore,
- boolean secure, boolean oneShotRun) {
+ boolean memoryMapping, int binariesInlineThreshold, boolean useBlobStore, int dsCacheInMB, int syncInterval,
+ boolean shareBlobStore, boolean secure, boolean oneShotRun) {
return new OakRepositoryFixture(OakFixture.getSegmentTarWithColdStandby(base, maxFileSizeMB, cacheSizeMB,
- memoryMapping, useBlobStore, dsCacheInMB, syncInterval, shareBlobStore, secure, oneShotRun));
+ memoryMapping, binariesInlineThreshold, useBlobStore, dsCacheInMB, syncInterval, shareBlobStore, secure, oneShotRun));
}
public static RepositoryFixture getSegmentTarWithAwsSegmentStore(final File base, final String awsBucketName,
final String awsRootPath, final String awsJournalTableName, final String awsLockTableName,
- final int maxFileSizeMB, final int cacheSizeMB, final boolean useBlobStore, final int dsCacheInMB) {
+ final int maxFileSizeMB, final int cacheSizeMB, final int binariesInlineThreshold, final boolean useBlobStore,
+ final int dsCacheInMB) {
return new OakRepositoryFixture(OakFixture.getSegmentTarWithAwsSegmentStore(base, awsBucketName, awsRootPath,
- awsJournalTableName, awsLockTableName, maxFileSizeMB, cacheSizeMB, useBlobStore, dsCacheInMB));
+ awsJournalTableName, awsLockTableName, maxFileSizeMB, cacheSizeMB, binariesInlineThreshold, useBlobStore, dsCacheInMB));
}
public static RepositoryFixture getSegmentTarWithAzureSegmentStore(final File base, final String azureConnectionString,
final String azureContainerName, final String azureRootPath,
- final int maxFileSizeMB, final int cacheSizeMB, final boolean useBlobStore,
+ final int maxFileSizeMB, final int cacheSizeMB,
+ final int binariesInlineThreshold, final boolean useBlobStore,
final int dsCacheInMB) {
return new OakRepositoryFixture(OakFixture.getSegmentTarWithAzureSegmentStore(base, azureConnectionString,
- azureContainerName, azureRootPath, maxFileSizeMB, cacheSizeMB, useBlobStore, dsCacheInMB));
+ azureContainerName, azureRootPath, maxFileSizeMB, cacheSizeMB, binariesInlineThreshold, useBlobStore, dsCacheInMB));
}
public static RepositoryFixture getCompositeStore(File base, int maxFileSizeMB, int cacheSizeMB,
- final boolean memoryMapping) {
+ final boolean memoryMapping, int binariesInlineThreshold) {
return new OakRepositoryFixture(OakFixture.getCompositeStore(OakFixture.OAK_COMPOSITE_STORE,
- base, maxFileSizeMB, cacheSizeMB, memoryMapping));
+ base, maxFileSizeMB, cacheSizeMB, memoryMapping, binariesInlineThreshold));
}
public static RepositoryFixture getCompositeMemoryStore() {
diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java
index 6a2a1316d6..470e6da8df 100644
--- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java
+++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java
@@ -81,6 +81,7 @@ public class SegmentTarFixture extends OakFixture {
private int maxFileSize;
private int segmentCacheSize;
private boolean memoryMapping;
+ public int binariesInlineThreshold;
private boolean useBlobStore;
private int dsCacheSize;
@@ -116,6 +117,11 @@ public class SegmentTarFixture extends OakFixture {
this.memoryMapping = memoryMapping;
return this;
}
+
+ public SegmentTarFixtureBuilder withBinariesInlineThreshold(int binariesInlineThreshold) {
+ this.binariesInlineThreshold = binariesInlineThreshold;
+ return this;
+ }
public SegmentTarFixtureBuilder withBlobStore(boolean useBlobStore) {
this.useBlobStore = useBlobStore;
@@ -151,6 +157,7 @@ public class SegmentTarFixture extends OakFixture {
private final int maxFileSize;
private final int segmentCacheSize;
private final boolean memoryMapping;
+ private final int binariesInlineThreshold;
private final boolean useBlobStore;
private final int dsCacheSize;
@@ -197,6 +204,7 @@ public class SegmentTarFixture extends OakFixture {
this.maxFileSize = builder.maxFileSize;
this.segmentCacheSize = builder.segmentCacheSize;
this.memoryMapping = builder.memoryMapping;
+ this.binariesInlineThreshold = builder.binariesInlineThreshold;
this.useBlobStore = builder.useBlobStore;
this.dsCacheSize = builder.dsCacheSize;
@@ -296,7 +304,9 @@ public class SegmentTarFixture extends OakFixture {
fileStoreBuilder.withBlobStore(blobStore);
}
- FileStore fs = fileStoreBuilder.build();
+ FileStore fs = fileStoreBuilder
+ .withBinariesInlineThreshold(binariesInlineThreshold)
+ .build();
Oak oak = newOak(SegmentNodeStoreBuilders.builder(fs).build());
if (blobStore != null) {
oak.getWhiteboard()
@@ -346,6 +356,7 @@ public class SegmentTarFixture extends OakFixture {
.withSegmentCacheSize(segmentCacheSize)
.withMemoryMapping(memoryMapping)
.withStrictVersionCheck(true)
+ .withBinariesInlineThreshold(binariesInlineThreshold)
.build();
if (withColdStandby) {