You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ng...@apache.org on 2020/03/27 07:44:46 UTC
svn commit: r1875760 [1/5] - in /jackrabbit/oak/trunk: ./
oak-benchmarks-lucene/ oak-benchmarks-lucene/src/
oak-benchmarks-lucene/src/main/ oak-benchmarks-lucene/src/main/java/
oak-benchmarks-lucene/src/main/java/org/
oak-benchmarks-lucene/src/main/jav...
Author: ngupta
Date: Fri Mar 27 07:44:43 2020
New Revision: 1875760
URL: http://svn.apache.org/viewvc?rev=1875760&view=rev
Log:
OAK-8962 | Restructuring oak-benchmarks to avoid lucene/solr/elastic
Added:
jackrabbit/oak/trunk/oak-benchmarks-lucene/
jackrabbit/oak/trunk/oak-benchmarks-lucene/pom.xml
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneBenchmarkRunner.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextSearchTest.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFTSeparated.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFullTextTest.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/util/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/util/OakLuceneIndexUtils.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkLuceneCommand.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneMain.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneScalabilityCommand.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/LuceneScalabilityRunner.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/AggregateNodeSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/FacetSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/FormatSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/FullTextSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/LastModifiedSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/MultiFilterOrderByKeysetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/MultiFilterOrderByOffsetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/MultiFilterOrderBySearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/MultiFilterSplitOrderByKeysetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/MultiFilterSplitOrderByOffsetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/MultiFilterSplitOrderBySearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/NodeTypeSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/OrderByDate.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/OrderByKeysetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/OrderByOffsetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/OrderBySearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/PaginationEnabledSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/SearchScalabilityBenchmark.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/SplitOrderByKeysetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/SplitOrderByOffsetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/SplitOrderBySearcher.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/suites/
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/suites/ScalabilityBlobSearchSuite.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/suites/ScalabilityNodeRelationshipSuite.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/suites/ScalabilityNodeSuite.java
jackrabbit/oak/trunk/oak-benchmarks-solr/
jackrabbit/oak/trunk/oak-benchmarks-solr/pom.xml
jackrabbit/oak/trunk/oak-benchmarks-solr/src/
jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/
jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/
jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/
jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/
jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/
jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/
jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/benchmark/
jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSolrSearchTest.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkOptions.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/ScalabilityOptions.java
Removed:
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSolrSearchTest.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFTSeparated.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFullTextTest.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/AggregateNodeSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/FacetSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/FormatSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/FullTextSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/LastModifiedSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/MultiFilterOrderByKeysetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/MultiFilterOrderByOffsetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/MultiFilterOrderBySearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/MultiFilterSplitOrderByKeysetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/MultiFilterSplitOrderByOffsetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/MultiFilterSplitOrderBySearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/NodeTypeSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/OrderByDate.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/OrderByKeysetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/OrderByOffsetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/OrderBySearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/PaginationEnabledSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/SearchScalabilityBenchmark.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/SplitOrderByKeysetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/SplitOrderByOffsetPageSearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/SplitOrderBySearcher.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/suites/ScalabilityBlobSearchSuite.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/suites/ScalabilityNodeRelationshipSuite.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/suites/ScalabilityNodeSuite.java
Modified:
jackrabbit/oak/trunk/oak-benchmarks/pom.xml
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AbstractTest.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSearchTest.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/util/OakIndexUtils.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scalability/ScalabilityRunner.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneInitializerHelper.java
jackrabbit/oak/trunk/pom.xml
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/pom.xml?rev=1875760&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/pom.xml (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/pom.xml Fri Mar 27 07:44:43 2020
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <parent>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-parent</artifactId>
+ <version>1.27-SNAPSHOT</version>
+ <relativePath>../oak-parent/pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>oak-benchmarks-lucene</artifactId>
+
+ <properties>
+ <skip.deployment>true</skip.deployment>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-benchmarks</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-lucene</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jetbrains</groupId>
+ <artifactId>annotations</artifactId>
+ </dependency>
+ </dependencies>
+
+
+</project>
\ No newline at end of file
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java?rev=1875760&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java Fri Mar 27 07:44:43 2020
@@ -0,0 +1,590 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.benchmark;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Queue;
+import java.util.Random;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+
+import javax.jcr.Node;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.MoreExecutors;
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.api.jmx.IndexStatsMBean;
+import org.apache.jackrabbit.oak.fixture.JcrCreator;
+import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
+import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoServiceImpl;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
+import org.apache.jackrabbit.oak.plugins.index.IndexPathServiceImpl;
+import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexObserver;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner;
+import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReaderFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder.PropertyRule;
+import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
+import org.apache.jackrabbit.oak.spi.commit.BackgroundObserver;
+import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer;
+import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
+import org.apache.jackrabbit.oak.spi.mount.Mounts;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.spi.whiteboard.Registration;
+import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
+import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.DECLARING_NODE_TYPES;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
+import static org.apache.jackrabbit.oak.spi.nodetype.NodeTypeConstants.NT_OAK_UNSTRUCTURED;
+import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.scheduleWithFixedDelay;
+
+public class HybridIndexTest extends AbstractTest<HybridIndexTest.TestContext> {
+ enum Status {
+ NONE, STARTING, STARTED, STOPPING, STOPPED, ABORTED;
+
+ private int count;
+
+ public void inc(){
+ count++;
+ }
+
+ public int count(){
+ return count;
+ }
+
+ public Status next(){
+ Status[] ss = values();
+ if (ordinal() == ss.length - 1){
+ return ss[0];
+ }
+ return ss[ordinal() + 1];
+ }
+ }
+
+ private final Random random = new Random(42); //fixed seed
+ private String indexedPropName = "foo";
+ private int nodesPerIteration = Status.values().length;
+ private int numOfIndexes = Integer.getInteger("numOfIndexes", 10);
+ private int refreshDeltaMillis = Integer.getInteger("refreshDeltaMillis", 1000);
+ private int asyncInterval = Integer.getInteger("asyncInterval", 5);
+ private int cleanerIntervalInSecs = Integer.getInteger("cleanerIntervalInSecs", 10);
+ private int queueSize = Integer.getInteger("queueSize", 1000);
+ private boolean hybridIndexEnabled = Boolean.getBoolean("hybridIndexEnabled");
+ private boolean dumpStats = Boolean.getBoolean("dumpStats");
+ private boolean useOakCodec = Boolean.parseBoolean(System.getProperty("useOakCodec", "true"));
+ private boolean syncIndexing = Boolean.parseBoolean(System.getProperty("syncIndexing", "false"));
+ private String indexingMode = System.getProperty("indexingMode", "nrt");
+
+ private boolean searcherEnabled = Boolean.parseBoolean(System.getProperty("searcherEnabled", "true"));
+ private File indexCopierDir;
+ private IndexCopier copier;
+ private NRTIndexFactory nrtIndexFactory;
+ private LuceneIndexProvider luceneIndexProvider;
+ private LuceneIndexEditorProvider luceneEditorProvider;
+ private DocumentQueue queue;
+ private LocalIndexObserver localIndexObserver;
+ private RepositoryInitializer indexInitializer = new PropertyIndexInitializer();
+ private TestContext defaultContext;
+ private final File workDir;
+ private Whiteboard whiteboard;
+ private Searcher searcher;
+ private Mutator mutator;
+ private final AtomicInteger indexedNodeCount = new AtomicInteger();
+ private List<TestContext> contexts = new ArrayList<>();
+ private final StatisticsProvider statsProvider;
+ private final Logger log = LoggerFactory.getLogger(getClass());
+ private final ExecutorService executorService = MoreExecutors.getExitingExecutorService(
+ (ThreadPoolExecutor) Executors.newFixedThreadPool(5));
+ private final List<Registration> regs = new ArrayList<>();
+ private BackgroundObserver backgroundObserver;
+
+
+ public HybridIndexTest(File workDir, StatisticsProvider statsProvider) {
+ this.workDir = workDir;
+ this.statsProvider = statsProvider;
+ }
+
+ @Override
+ protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
+ if (fixture instanceof OakRepositoryFixture) {
+ return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() {
+ @Override
+ public Jcr customize(Oak oak) {
+ Jcr jcr = new Jcr(oak);
+ whiteboard = oak.getWhiteboard();
+ prepareLuceneIndexer(workDir, getNodeStore(oak));
+
+ backgroundObserver = new BackgroundObserver(luceneIndexProvider, executorService, 5);
+
+ jcr.with((QueryIndexProvider) luceneIndexProvider)
+ .with(backgroundObserver)
+ .with(luceneEditorProvider)
+ .with(new NodeTypeIndexFixerInitializer());
+
+ if (hybridIndexEnabled) {
+ jcr.with(localIndexObserver);
+ indexInitializer = new LuceneIndexInitializer();
+ }
+
+ jcr.with(indexInitializer);
+
+ //Configure the default global fulltext index as it impacts
+ //both pure property index based setup and nrt based
+ //So more closer to real world
+ jcr.with(new LuceneFullTextInitializer());
+
+ //Async indexing is enabled for both property and lucene
+ //as for property it relies on counter index
+ oak.withAsyncIndexing("async", asyncInterval);
+ return jcr;
+ }
+ });
+ }
+ return super.createRepository(fixture);
+ }
+
+ @Override
+ public void beforeSuite() throws Exception {
+ if (hybridIndexEnabled) {
+ runAsyncIndex();
+ }
+ defaultContext = new TestContext();
+ contexts.add(defaultContext);
+ searcher = new Searcher();
+ mutator = new Mutator();
+
+ if (searcherEnabled) {
+ addBackgroundJob(searcher);
+ }
+
+ addBackgroundJob(mutator);
+ }
+
+ @Override
+ protected TestContext prepareThreadExecutionContext() throws RepositoryException {
+ TestContext ctx = new TestContext();
+ contexts.add(ctx);
+ return ctx;
+ }
+
+ @Override
+ protected void runTest() throws Exception {
+ runTest(defaultContext);
+ }
+
+ @Override
+ protected void runTest(TestContext ctx) throws Exception {
+ //Create tree in breadth first fashion with each node having 50 child
+ Node parent = ctx.session.getNode(ctx.paths.remove());
+ Status status = Status.NONE;
+ for (int i = 0; i < nodesPerIteration; i++) {
+ Node child = parent.addNode(nextNodeName());
+ child.setProperty(indexedPropName, status.name());
+ ctx.session.save();
+ ctx.paths.add(child.getPath());
+ indexedNodeCount.incrementAndGet();
+ status.inc();
+ status = status.next();
+ }
+ }
+
+ @Override
+ protected void disposeThreadExecutionContext(TestContext context) throws RepositoryException {
+ context.dispose();
+ }
+
+ @Override
+ protected void afterSuite() throws Exception {
+ //TODO This to avoid issue with Indexing still running post afterSuite call
+ //TO handle this properly we would need a callback after repository shutdown
+ //and before NodeStore teardown
+ getAsyncIndexUpdate().close();
+
+ if (backgroundObserver != null){
+ backgroundObserver.close();
+ }
+
+ int sleepCount = 0;
+ while (backgroundObserver.getMBean().getQueueSize()> 0 && ++sleepCount < 100) {
+ TimeUnit.MILLISECONDS.sleep(100);
+ }
+
+ for (Registration r : regs) {
+ r.unregister();
+ }
+
+ //Close hybrid stuff after async is closed
+ if (hybridIndexEnabled){
+ queue.close();
+ nrtIndexFactory.close();
+ }
+
+ if (indexCopierDir != null) {
+ FileUtils.deleteDirectory(indexCopierDir);
+ }
+ System.out.printf("numOfIndexes: %d, refreshDeltaMillis: %d, asyncInterval: %d, queueSize: %d , " +
+ "hybridIndexEnabled: %s, indexingMode: %s, useOakCodec: %s, cleanerIntervalInSecs: %d, " +
+ "syncIndexing: %s %n",
+ numOfIndexes, refreshDeltaMillis, asyncInterval, queueSize, hybridIndexEnabled,
+ indexingMode, useOakCodec, cleanerIntervalInSecs, syncIndexing);
+
+ if (dumpStats) {
+ dumpStats();
+ }
+ }
+
+ @Override
+ protected String[] statsNames() {
+ return new String[]{"Searcher", "Mutator", "Indexed"};
+ }
+
+ @Override
+ protected String[] statsFormats() {
+ return new String[]{"%8d", "%8d", "%8d"};
+ }
+
+ @Override
+ protected Object[] statsValues() {
+ return new Object[]{searcher.resultSize, mutator.mutationCount, indexedNodeCount.get()};
+ }
+
+ @Override
+ protected String comment() {
+ List<String> commentElements = new ArrayList<>();
+ if (hybridIndexEnabled){
+ commentElements.add(indexingMode);
+
+ if (useOakCodec){
+ commentElements.add("oakCodec");
+ }
+ if (syncIndexing) {
+ commentElements.add("sync");
+ }
+ } else {
+ commentElements.add("property");
+ }
+
+ commentElements.add("numIdxs:"+ numOfIndexes);
+ return Joiner.on(',').join(commentElements);
+ }
+
+ protected class TestContext {
+ final Session session = loginWriter();
+ final Queue<String> paths = new LinkedBlockingDeque<>();
+
+ final Node dump;
+
+ public TestContext() throws RepositoryException {
+ dump = session.getRootNode()
+ .addNode(nextNodeName(), NT_OAK_UNSTRUCTURED)
+ .addNode(nextNodeName(), NT_OAK_UNSTRUCTURED)
+ .addNode(nextNodeName(), NT_OAK_UNSTRUCTURED)
+ .addNode(nextNodeName(), NT_OAK_UNSTRUCTURED)
+ .addNode(nextNodeName(), NT_OAK_UNSTRUCTURED)
+ .addNode(nextNodeName(), NT_OAK_UNSTRUCTURED);
+ session.save();
+ paths.add(dump.getPath());
+ }
+
+ public void dispose() throws RepositoryException {
+ dump.remove();
+ session.logout();
+ }
+ }
+
+ private String randomStatus() {
+ Status status = Status.values()[random.nextInt(Status.values().length)];
+ status.inc();
+ return status.name();
+ }
+
+ private void prepareLuceneIndexer(File workDir, NodeStore nodeStore) {
+ try {
+ indexCopierDir = createTemporaryFolderIn(workDir);
+ copier = new IndexCopier(executorService, indexCopierDir, true);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ IndexPathService indexPathService = new IndexPathServiceImpl(nodeStore);
+ AsyncIndexInfoService asyncIndexInfoService = new AsyncIndexInfoServiceImpl(nodeStore);
+
+ nrtIndexFactory = new NRTIndexFactory(copier, Clock.SIMPLE,
+ TimeUnit.MILLISECONDS.toSeconds(refreshDeltaMillis), StatisticsProvider.NOOP);
+ MountInfoProvider mip = Mounts.defaultMountInfoProvider();
+ LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier);
+
+ IndexTracker tracker = new IndexTracker(indexReaderFactory, nrtIndexFactory);
+
+ luceneIndexProvider = new LuceneIndexProvider(tracker);
+ luceneEditorProvider = new LuceneIndexEditorProvider(copier,
+ tracker,
+ null, //extractedTextCache
+ null, //augmentorFactory
+ mip);
+
+ queue = new DocumentQueue(queueSize, tracker, executorService, statsProvider);
+ localIndexObserver = new LocalIndexObserver(queue, statsProvider);
+ luceneEditorProvider.setIndexingQueue(queue);
+
+ if (syncIndexing) {
+ PropertyIndexCleaner cleaner = new PropertyIndexCleaner(nodeStore, indexPathService, asyncIndexInfoService, statsProvider);
+ regs.add(scheduleWithFixedDelay(whiteboard, cleaner,
+ cleanerIntervalInSecs, true, true));
+ }
+
+
+ Thread.setDefaultUncaughtExceptionHandler((t, e) -> log.warn("Uncaught exception", e));
+ }
+
+ private void runAsyncIndex() {
+ checkNotNull(getAsyncIndexUpdate()).run();
+ }
+
+ private AsyncIndexUpdate getAsyncIndexUpdate() {
+ return (AsyncIndexUpdate)WhiteboardUtils.getService(whiteboard, Runnable.class, new Predicate<Runnable>() {
+ @Override
+ public boolean test(@Nullable Runnable input) {
+ return input instanceof AsyncIndexUpdate;
+ }
+ });
+ }
+
+ private void dumpStats() {
+ IndexStatsMBean indexStats = WhiteboardUtils.getService(whiteboard, IndexStatsMBean.class);
+ System.out.println(indexStats.getConsolidatedExecutionStats());
+ String queueSize = Arrays.toString(statsProvider.getStats().getTimeSeries("HYBRID_QUEUE_SIZE", false)
+ .getValuePerSecond());
+ System.out.println("Queue size - " + queueSize);
+ }
+
+ @SuppressWarnings("ResultOfMethodCallIgnored")
+ private static File createTemporaryFolderIn(File parentFolder) throws IOException {
+ File createdFolder = File.createTempFile("oak-", "", parentFolder);
+ createdFolder.delete();
+ createdFolder.mkdir();
+ return createdFolder;
+ }
+
+ private static NodeStore getNodeStore(Oak oak) {
+ try {
+ Field f = Oak.class.getDeclaredField("store");
+ f.setAccessible(true);
+ return (NodeStore) f.get(oak);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private class PropertyIndexInitializer implements RepositoryInitializer {
+
+ @Override
+ public void initialize(@NotNull NodeBuilder builder) {
+ NodeBuilder oakIndex = IndexUtils.getOrCreateOakIndex(builder);
+ addPropIndexDefn(oakIndex, indexedPropName);
+ for (int i = 0; i < numOfIndexes - 1; i++) {
+ addPropIndexDefn(oakIndex, indexedPropName + i);
+ }
+ }
+
+ private void addPropIndexDefn(NodeBuilder parent, String propName){
+ try {
+ NodeBuilder idx = IndexUtils.createIndexDefinition(parent, propName, false,
+ singleton(propName), null, "property", null);
+ if ( propName.equals(indexedPropName)) {
+ idx.setProperty("tags", singletonList("fooIndex"), Type.STRINGS);
+ }
+ } catch (RepositoryException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+ }
+
+ private class LuceneIndexInitializer implements RepositoryInitializer {
+ @Override
+ public void initialize(@NotNull NodeBuilder builder) {
+ NodeBuilder oakIndex = IndexUtils.getOrCreateOakIndex(builder);
+
+ IndexDefinitionBuilder defnBuilder = new IndexDefinitionBuilder();
+ defnBuilder.evaluatePathRestrictions();
+ defnBuilder.async("async", indexingMode, "async");
+ PropertyRule pr = defnBuilder.indexRule("nt:base").property(indexedPropName).propertyIndex();
+ if (syncIndexing) {
+ pr.sync();
+ }
+ if (useOakCodec) {
+ defnBuilder.codec("oakCodec");
+ }
+
+ for (int i = 0; i < numOfIndexes - 1; i++) {
+ defnBuilder.indexRule("nt:base").property(indexedPropName + i).propertyIndex();
+ }
+
+ oakIndex.setChildNode(indexedPropName, defnBuilder.build());
+ oakIndex.child(indexedPropName).setProperty("tags", singletonList("fooIndex"), Type.STRINGS);
+ }
+ }
+
+ private class LuceneFullTextInitializer implements RepositoryInitializer {
+ @Override
+ public void initialize(@NotNull NodeBuilder builder) {
+ NodeBuilder oakIndex = IndexUtils.getOrCreateOakIndex(builder);
+
+ IndexDefinitionBuilder defnBuilder = new IndexDefinitionBuilder();
+ defnBuilder.async("async", "async");
+ defnBuilder.codec("Lucene46");
+ defnBuilder.indexRule("nt:base")
+ .property(FulltextIndexConstants.REGEX_ALL_PROPS, true)
+ .nodeScopeIndex();
+ oakIndex.setChildNode("globalIndex", defnBuilder.build());
+ }
+ }
+
+ private class NodeTypeIndexFixerInitializer implements RepositoryInitializer {
+
+ @Override
+ public void initialize(@NotNull NodeBuilder builder) {
+ //Due to OAK-1150 currently all nodes get indexed
+ //With explicit list on those nodes would be indexed
+ NodeBuilder nodetype = builder.getChildNode("oak:index").getChildNode("nodetype");
+ if (nodetype.exists()) {
+ List<String> nodetypes = Lists.newArrayList();
+ if (nodetype.hasProperty(DECLARING_NODE_TYPES)){
+ nodetypes = Lists.newArrayList(nodetype.getProperty(DECLARING_NODE_TYPES).getValue(Type.STRINGS));
+ }
+
+ if (nodetypes.isEmpty()) {
+ nodetypes.add(INDEX_DEFINITIONS_NODE_TYPE);
+ nodetypes.add("rep:Authorizable");
+ nodetype.setProperty(DECLARING_NODE_TYPES, nodetypes, Type.NAMES);
+ nodetype.setProperty(IndexConstants.REINDEX_PROPERTY_NAME, true);
+ }
+ }
+
+ //Disable counter index to disable traversal
+ NodeBuilder counter = builder.getChildNode("oak:index").getChildNode("counter");
+ if (counter.exists()) {
+ counter.setProperty("type", "disabled");
+ }
+ }
+ }
+
+ private class Searcher implements Runnable {
+ final Session session = loginWriter();
+ int resultSize = 0;
+ @Override
+ public void run() {
+ try{
+ run0();
+ } catch (RepositoryException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void run0() throws RepositoryException {
+ session.refresh(false);
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Query q = qm.createQuery("select * from [nt:base] where [" + indexedPropName + "] = $status " +
+ "option(index tag fooIndex)", Query.JCR_SQL2);
+ q.bindValue("status", session.getValueFactory().createValue(randomStatus()));
+ QueryResult result = q.execute();
+
+ //With property index at time traversing index wins (somehow reporting lower cost)
+ //and that leads to warning. So limit the iterator size
+ resultSize += Iterators.size(Iterators.limit(result.getNodes(), 500));
+ }
+ }
+
+ private class Mutator implements Runnable {
+ final Session session = loginWriter();
+ int mutationCount = 0;
+ @Override
+ public void run() {
+ try{
+ run0();
+ } catch (RepositoryException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void run0() throws RepositoryException {
+ TestContext ctx = contexts.get(random.nextInt(contexts.size()));
+ String path = ctx.paths.peek();
+ session.refresh(false);
+ if (path != null){
+ Node node = session.getNode(path);
+ if(node.hasProperty(indexedPropName)){
+ String value = node.getProperty(indexedPropName).getString();
+ String newValue = Status.valueOf(value).next().name();
+ node.setProperty(indexedPropName, newValue);
+ session.save();
+ mutationCount++;
+ }
+ }
+ }
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneBenchmarkRunner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneBenchmarkRunner.java?rev=1875760&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneBenchmarkRunner.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneBenchmarkRunner.java Fri Mar 27 07:44:43 2020
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+
+public class LuceneBenchmarkRunner extends BenchmarkRunner {
+
+ public static void main(String[] args) throws Exception {
+ statsProvider = options.has(benchmarkOptions.getMetrics()) ? getStatsProvider() : StatisticsProvider.NOOP;
+ initOptionSet(args);
+ BenchmarkRunner.addToBenchMarkList(
+ Arrays.asList(
+ new LuceneFullTextSearchTest(
+ benchmarkOptions.getWikipedia().value(options),
+ benchmarkOptions.getFlatStructure().value(options),
+ benchmarkOptions.getReport().value(options),
+ benchmarkOptions.getWithStorage().value(options)),
+ new LucenePropertyFullTextTest(
+ benchmarkOptions.getWikipedia().value(options),
+ benchmarkOptions.getFlatStructure().value(options),
+ benchmarkOptions.getReport().value(options), benchmarkOptions.getWithStorage().value(options)),
+ new LucenePropertyFTSeparated(
+ benchmarkOptions.getWikipedia().value(options),
+ benchmarkOptions.getFlatStructure().value(options),
+ benchmarkOptions.getReport().value(options), benchmarkOptions.getWithStorage().value(options)),
+ new HybridIndexTest(benchmarkOptions.getBase().value(options), statsProvider)
+ )
+ );
+
+ BenchmarkRunner.main(args);
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextSearchTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextSearchTest.java?rev=1875760&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextSearchTest.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextSearchTest.java Fri Mar 27 07:44:43 2020
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.fixture.JcrCreator;
+import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
+import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneInitializerHelper;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+
+import javax.jcr.Repository;
+import java.io.File;
+import java.io.IOException;
+
+public class LuceneFullTextSearchTest extends FullTextSearchTest {
+
+ private final boolean disableCopyOnRead = Boolean.getBoolean("disableCopyOnRead");
+
+ public LuceneFullTextSearchTest(File dump, boolean flat, boolean doReport, Boolean storageEnabled) {
+ super(dump, flat, doReport, storageEnabled);
+ }
+
+
+ @Override
+ protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
+ if (fixture instanceof OakRepositoryFixture) {
+ return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() {
+ @Override
+ public Jcr customize(Oak oak) {
+ LuceneIndexProvider provider = createLuceneIndexProvider();
+ oak.with((QueryIndexProvider) provider)
+ .with((Observer) provider)
+ .with(new LuceneIndexEditorProvider())
+ .with(new LuceneInitializerHelper("luceneGlobal", storageEnabled));
+ return new Jcr(oak);
+ }
+ });
+ }
+ return super.createRepository(fixture);
+ }
+
+ private LuceneIndexProvider createLuceneIndexProvider() {
+ if (!disableCopyOnRead) {
+ try {
+ IndexCopier copier = new IndexCopier(executorService, indexCopierDir, true);
+ return new LuceneIndexProvider(copier);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return new LuceneIndexProvider();
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFTSeparated.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFTSeparated.java?rev=1875760&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFTSeparated.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFTSeparated.java Fri Mar 27 07:44:43 2020
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+import static com.google.common.collect.ImmutableSet.of;
+
+import java.io.File;
+
+import javax.jcr.Repository;
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.fixture.JcrCreator;
+import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
+import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneInitializerHelper;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+
+/**
+ * same as {@link LucenePropertyFullTextTest} but will initialise a repository where the global
+ * full-text runs on a separate thread from lucene property.
+ */
+public class LucenePropertyFTSeparated extends LucenePropertyFullTextTest {
+
+ public LucenePropertyFTSeparated(final File dump,
+ final boolean flat,
+ final boolean doReport,
+ final Boolean storageEnabled) {
+ super(dump, flat, doReport, storageEnabled);
+ currentTest = this.getClass().getSimpleName();
+ }
+
+ @Override
+ protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
+ if (fixture instanceof OakRepositoryFixture) {
+ currentFixture = fixture.toString();
+ return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() {
+ @Override
+ public Jcr customize(Oak oak) {
+ LuceneIndexProvider provider = new LuceneIndexProvider();
+ oak.with((QueryIndexProvider) provider)
+ .with((Observer) provider)
+ .with(new LuceneIndexEditorProvider())
+ .with(
+ (new LuceneInitializerHelper("luceneGlobal", storageEnabled))
+ .async("async-slow"))
+ // the WikipediaImporter set a property `title`
+ .with(new LucenePropertyInitialiser("luceneTitle", of("title")))
+ .withAsyncIndexing("async", 5)
+ .withAsyncIndexing("async-slow", 5);
+ return new Jcr(oak);
+ }
+ });
+ }
+ return super.createRepository(fixture);
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFullTextTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFullTextTest.java?rev=1875760&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFullTextTest.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFullTextTest.java Fri Mar 27 07:44:43 2020
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.ImmutableSet.of;
+import static org.apache.jackrabbit.oak.api.Type.BOOLEAN;
+import static org.apache.jackrabbit.oak.api.Type.LONG;
+import static org.apache.jackrabbit.oak.api.Type.NAME;
+import static org.apache.jackrabbit.oak.api.Type.STRING;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.ASYNC_PROPERTY_NAME;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE;
+import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.COMPAT_MODE;
+import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INDEX_RULES;
+import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NAME;
+import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NODE;
+import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_PROPERTY_INDEX;
+
+import java.io.File;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.ValueFactory;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.RowIterator;
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.api.Tree;
+import org.apache.jackrabbit.oak.benchmark.wikipedia.WikipediaImport;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.fixture.JcrCreator;
+import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
+import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneInitializerHelper;
+import org.apache.jackrabbit.oak.plugins.tree.factories.TreeFactory;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * Perform a benchmark on how long it takes for an ingested item to be available in a Lucene
+ * Property index when indexed in conjunction with a Global full-text lucene (same thread). It makes
+ * use of the {@link WikipediaImport} to use a Wikipedia dump for content injestion.
+ * </p>
+ * <p>
+ * Suggested dump:
+ * <a href="https://dumps.wikimedia.org/enwiki/20150403/enwiki-20150403-pages-articles.xml.bz2">https://dumps.wikimedia.org/enwiki/20150403/enwiki-20150403-pages-articles.xml.bz2</a>
+ * </p>
+ * <p>
+ * Usage example:
+ * </p>
+ *
+ * <pre>
+ * java -Druntime=900 -Dlogback.configurationFile=logback-benchmark.xml \
+ * -jar ~/.m2/repository/org/apache/jackrabbit/oak-run/1.4-SNAPSHOT/oak-run-1.4-SNAPSHOT.jar \
+ * benchmark --wikipedia enwiki-20150403-pages-articles.xml.bz2 \
+ * --base ~/tmp/oak/ LucenePropertyFullTextTest Oak-Tar Oak-Mongo
+ * </pre>
+ * <p>
+ * it will run the benchmark for 15 minutes against TarNS and MongoNS.
+ * </p>
+ */
+public class LucenePropertyFullTextTest extends AbstractTest<LucenePropertyFullTextTest.TestContext> {
+ private static final Logger LOG = LoggerFactory.getLogger(LucenePropertyFullTextTest.class);
+ private WikipediaImport importer;
+ private Thread asyncImporter;
+ private boolean benchmarkCompleted, importerCompleted;
+ Boolean storageEnabled;
+ String currentFixture, currentTest;
+
+ /**
+ * context used across the tests
+ */
+ class TestContext {
+ final Session session = loginWriter();
+ final String title;
+
+ public TestContext(@NotNull final String title) {
+ this.title = checkNotNull(title);
+ }
+ }
+
+ /**
+ * helper class to initialise the Lucene Property index definition
+ */
+ static class LucenePropertyInitialiser implements RepositoryInitializer {
+ private String name;
+ private Set<String> properties;
+
+ public LucenePropertyInitialiser(@NotNull final String name,
+ @NotNull final Set<String> properties) {
+ this.name = checkNotNull(name);
+ this.properties = checkNotNull(properties);
+ }
+
+ private boolean isAlreadyThere(@NotNull final NodeBuilder root) {
+ return checkNotNull(root).hasChildNode(INDEX_DEFINITIONS_NAME) &&
+ root.getChildNode(INDEX_DEFINITIONS_NAME).hasChildNode(name);
+ }
+
+ @Override
+ public void initialize(final NodeBuilder builder) {
+ if (!isAlreadyThere(builder)) {
+ Tree t = TreeFactory.createTree(builder.child(INDEX_DEFINITIONS_NAME));
+ t = t.addChild(name);
+ t.setProperty("jcr:primaryType", INDEX_DEFINITIONS_NODE_TYPE, NAME);
+ t.setProperty(COMPAT_MODE, 2L, LONG);
+ t.setProperty(TYPE_PROPERTY_NAME, TYPE_LUCENE, STRING);
+ t.setProperty(ASYNC_PROPERTY_NAME, "async", STRING);
+ t.setProperty(REINDEX_PROPERTY_NAME, true);
+
+ t = t.addChild(INDEX_RULES);
+ t.setOrderableChildren(true);
+ t.setProperty("jcr:primaryType", "nt:unstructured", NAME);
+
+ t = t.addChild("nt:base");
+
+ Tree propnode = t.addChild(PROP_NODE);
+ propnode.setOrderableChildren(true);
+ propnode.setProperty("jcr:primaryType", "nt:unstructured", NAME);
+
+ for (String p : properties) {
+ Tree t1 = propnode.addChild(PathUtils.getName(p));
+ t1.setProperty(PROP_PROPERTY_INDEX, true, BOOLEAN);
+ t1.setProperty(PROP_NAME, p);
+ }
+ }
+ }
+ }
+
+ /**
+ * reference to the last added title. Used for looking up with queries.
+ */
+ private AtomicReference<String> lastTitle = new AtomicReference<String>();
+
+ public LucenePropertyFullTextTest(final File dump,
+ final boolean flat,
+ final boolean doReport,
+ final Boolean storageEnabled) {
+ this.importer = new WikipediaImport(dump, flat, doReport) {
+
+ @Override
+ protected void pageAdded(String title, String text) {
+ LOG.trace("Setting title: {}", title);
+ lastTitle.set(title);
+ }
+ };
+ this.storageEnabled = storageEnabled;
+ this.currentTest = this.getClass().getSimpleName();
+ }
+
+ @Override
+ protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
+ if (fixture instanceof OakRepositoryFixture) {
+ currentFixture = fixture.toString();
+ return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() {
+ @Override
+ public Jcr customize(Oak oak) {
+ LuceneIndexProvider provider = new LuceneIndexProvider();
+ oak.with((QueryIndexProvider) provider)
+ .with((Observer) provider)
+ .with(new LuceneIndexEditorProvider())
+ .with((new LuceneInitializerHelper("luceneGlobal", storageEnabled)).async())
+ // the WikipediaImporter set a property `title`
+ .with(new LucenePropertyInitialiser("luceneTitle", of("title")))
+ .withAsyncIndexing("async", 5);
+ return new Jcr(oak);
+ }
+ });
+ }
+ return super.createRepository(fixture);
+ }
+
+ @Override
+ protected void beforeSuite() throws Exception {
+ LOG.debug("beforeSuite() - {} - {}", currentFixture, currentTest);
+ benchmarkCompleted = false;
+ importerCompleted = false;
+ asyncImporter = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ importer.importWikipedia(loginWriter());
+ } catch (Exception e) {
+ LOG.error("Error while importing the dump. Trying to halt everything.", e);
+ importerCompleted = true;
+ } finally {
+ if (!benchmarkCompleted) {
+ importerCompleted = true;
+ issueHaltRequest("Wikipedia import completed.");
+ }
+ }
+ }
+ });
+ asyncImporter.start();
+
+ // allowing the async index to catch up.
+ TimeUnit.SECONDS.sleep(10);
+ }
+
+ @Override
+ protected void afterSuite() throws Exception {
+ LOG.debug("afterSuite() - {} - {}", currentFixture, currentTest);
+ asyncImporter.join();
+ }
+
+ @Override
+ protected void runTest() throws Exception {
+ if (lastTitle.get() == null) {
+ return;
+ }
+ runTest(new TestContext(lastTitle.get()));
+ }
+
+ @Override
+ protected void runTest(final TestContext ec) throws Exception {
+ if (importerCompleted) {
+ return;
+ }
+ final long maxWait = TimeUnit.MINUTES.toMillis(5);
+ final long waitUnit = 50;
+ long sleptSoFar = 0;
+
+ while (!performQuery(ec) && sleptSoFar < maxWait) {
+ LOG.trace("title '{}' not found. Waiting and retry. sleptSoFar: {}ms", ec.title,
+ sleptSoFar);
+ sleptSoFar += waitUnit;
+ TimeUnit.MILLISECONDS.sleep(waitUnit);
+ }
+
+ if (sleptSoFar < maxWait) {
+ // means we exited the loop as we found it.
+ LOG.info("{} - {} - title '{}' found with a wait/try of {}ms", currentFixture,
+ currentTest, ec.title, sleptSoFar);
+ } else {
+ LOG.warn("{} - {} - title '{}' timed out with a way/try of {}ms.", currentFixture,
+ currentTest, ec.title, sleptSoFar);
+ }
+ }
+
+ private boolean performQuery(@NotNull final TestContext ec) throws RepositoryException {
+ QueryManager qm = ec.session.getWorkspace().getQueryManager();
+ ValueFactory vf = ec.session.getValueFactory();
+ Query q = qm.createQuery("SELECT * FROM [nt:base] WHERE [title] = $title", Query.JCR_SQL2);
+ q.bindValue("title", vf.createValue(ec.title));
+ LOG.trace("statement: {} - title: {}", q.getStatement(), ec.title);
+ RowIterator rows = q.execute().getRows();
+ if (rows.hasNext()) {
+ rows.nextRow().getPath();
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ protected void issueHaltChildThreads() {
+ if (!importerCompleted) {
+ LOG.info("benchmark completed. Issuing an halt for the importer");
+ benchmarkCompleted = true;
+ this.importer.issueHaltImport();
+ }
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/util/OakLuceneIndexUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/util/OakLuceneIndexUtils.java?rev=1875760&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/util/OakLuceneIndexUtils.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/util/OakLuceneIndexUtils.java Fri Mar 27 07:44:43 2020
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark.util;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import org.apache.jackrabbit.commons.JcrUtils;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
+import org.apache.jackrabbit.oak.spi.nodetype.NodeTypeConstants;
+
+import javax.jcr.Node;
+import javax.jcr.PropertyType;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import java.util.List;
+import java.util.Map;
+
+public class OakLuceneIndexUtils {
+
+ /**
+ * Helper method to create or update a lucene property index definition.
+ *
+ * @param session the session
+ * @param indexDefinitionName the name of the node for the index definition
+ * @param propertyNames the list of properties to index
+ * @param type the types of the properties in order of the properties
+ * @param orderedPropsMap the ordered props and its properties
+ * @param persistencePath the path if the persistence=file (default is repository)
+ * @return the node just created
+ * @throws RepositoryException the repository exception
+ */
+ public static Node luceneIndexDefinition(Session session, String indexDefinitionName,
+ String async, String[] propertyNames, String[] type,
+ Map<String, Map<String, String>> orderedPropsMap, String persistencePath)
+ throws RepositoryException {
+
+ Node root = session.getRootNode();
+ Node indexDefRoot = JcrUtils.getOrAddNode(root, IndexConstants.INDEX_DEFINITIONS_NAME,
+ NodeTypeConstants.NT_UNSTRUCTURED);
+
+ Node indexDef = JcrUtils.getOrAddNode(indexDefRoot, indexDefinitionName,
+ IndexConstants.INDEX_DEFINITIONS_NODE_TYPE);
+
+ indexDef.setProperty(IndexConstants.TYPE_PROPERTY_NAME, LuceneIndexConstants.TYPE_LUCENE);
+ indexDef.setProperty(FulltextIndexConstants.FULL_TEXT_ENABLED, false);
+ if (async != null) {
+ indexDef.setProperty(IndexConstants.ASYNC_PROPERTY_NAME, async);
+ }
+ // Set indexed property names
+ indexDef.setProperty(FulltextIndexConstants.INCLUDE_PROPERTY_NAMES, propertyNames,
+ PropertyType.NAME);
+
+ Node propsNode = JcrUtils.getOrAddNode(indexDef, FulltextIndexConstants.PROP_NODE);
+ for (int i = 0; i < propertyNames.length; i++) {
+ Node propNode =
+ JcrUtils.getOrAddNode(propsNode, propertyNames[i], NodeTypeConstants.NT_OAK_UNSTRUCTURED);
+ propNode.setProperty(FulltextIndexConstants.PROP_TYPE, type[i]);
+ }
+
+ // Set ordered property names
+ if ((orderedPropsMap != null) && !orderedPropsMap.isEmpty()) {
+ List<String> orderedProps = Lists.newArrayList();
+ for (Map.Entry<String, Map<String, String>> orderedPropEntry : orderedPropsMap
+ .entrySet()) {
+ Node propNode = JcrUtils.getOrAddNode(propsNode, orderedPropEntry.getKey(),
+ NodeTypeConstants.NT_OAK_UNSTRUCTURED);
+ propNode.setProperty(FulltextIndexConstants.PROP_TYPE,
+ orderedPropEntry.getValue().get(FulltextIndexConstants.PROP_TYPE));
+ orderedProps.add(orderedPropEntry.getKey());
+ }
+ if (!orderedProps.isEmpty()) {
+ indexDef.setProperty(FulltextIndexConstants.ORDERED_PROP_NAMES,
+ orderedProps.toArray(new String[orderedProps.size()]),
+ PropertyType.NAME);
+ }
+ }
+
+ // Set file persistence if specified
+ if (!Strings.isNullOrEmpty(persistencePath)) {
+ indexDef.setProperty(FulltextIndexConstants.PERSISTENCE_NAME,
+ FulltextIndexConstants.PERSISTENCE_FILE);
+ indexDef.setProperty(FulltextIndexConstants.PERSISTENCE_PATH,
+ persistencePath);
+ }
+ session.save();
+
+ return indexDef;
+ }
+
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkLuceneCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkLuceneCommand.java?rev=1875760&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkLuceneCommand.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkLuceneCommand.java Fri Mar 27 07:44:43 2020
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+
+import org.apache.jackrabbit.oak.benchmark.LuceneBenchmarkRunner;
+import org.apache.jackrabbit.oak.run.commons.Command;
+
+public class BenchmarkLuceneCommand implements Command {
+
+ @Override
+ public void execute(String... args) throws Exception {
+ LuceneBenchmarkRunner.main(args);
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneMain.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneMain.java?rev=1875760&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneMain.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneMain.java Fri Mar 27 07:44:43 2020
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.run.commons.Modes;
+import org.apache.jackrabbit.oak.run.commons.Utils;
+
+import static java.util.Arrays.copyOfRange;
+
+public class LuceneMain {
+
+ private static final Modes MODES = new Modes(ImmutableMap.<String, Command>of(
+ "benchmark", new BenchmarkLuceneCommand(),
+ "scalability", new ScalabilityCommand()
+ ));
+
+ private LuceneMain() {
+ // Prevent instantiation.
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ Utils.printProductInfo(
+ args,
+ Main.class.getResourceAsStream("/META-INF/maven/org.apache.jackrabbit/oak-benchmarks-lucene/pom.properties")
+ );
+
+ Command c = MODES.getCommand("benchmark");
+ if (args.length > 0) {
+ c = MODES.getCommand(args[0]);
+
+ if (c == null) {
+ c = MODES.getCommand("benchmark");
+ }
+
+ args = copyOfRange(args, 1, args.length);
+ }
+
+ c.execute(args);
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneScalabilityCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneScalabilityCommand.java?rev=1875760&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneScalabilityCommand.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneScalabilityCommand.java Fri Mar 27 07:44:43 2020
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+
+import org.apache.jackrabbit.oak.scalability.LuceneScalabilityRunner;
+
+public class LuceneScalabilityCommand extends ScalabilityCommand {
+
+ @Override
+ public void execute(String... args) throws Exception {
+ LuceneScalabilityRunner.main(args);
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/LuceneScalabilityRunner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/LuceneScalabilityRunner.java?rev=1875760&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/LuceneScalabilityRunner.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/LuceneScalabilityRunner.java Fri Mar 27 07:44:43 2020
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.scalability;
+
+import org.apache.jackrabbit.oak.benchmark.util.Date;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.AggregateNodeSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.ConcurrentReader;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.ConcurrentWriter;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.FacetSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.FormatSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.FullTextSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.LastModifiedSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.MultiFilterOrderByKeysetPageSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.MultiFilterOrderByOffsetPageSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.MultiFilterOrderBySearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.MultiFilterSplitOrderByKeysetPageSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.MultiFilterSplitOrderByOffsetPageSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.MultiFilterSplitOrderBySearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.NodeTypeSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.OrderByDate;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.OrderByKeysetPageSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.OrderByOffsetPageSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.OrderBySearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.SplitOrderByKeysetPageSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.SplitOrderByOffsetPageSearcher;
+import org.apache.jackrabbit.oak.scalability.benchmarks.search.SplitOrderBySearcher;
+import org.apache.jackrabbit.oak.scalability.suites.ScalabilityBlobSearchSuite;
+import org.apache.jackrabbit.oak.scalability.suites.ScalabilityNodeRelationshipSuite;
+import org.apache.jackrabbit.oak.scalability.suites.ScalabilityNodeSuite;
+
+import java.util.Arrays;
+
+public class LuceneScalabilityRunner extends ScalabilityRunner {
+
+ public static void main(String[] args) throws Exception {
+ initOptionSet(args);
+ ScalabilityRunner.addToScalabilitySuiteList(
+ Arrays.asList(
+ new ScalabilityBlobSearchSuite(scalabilityOptions.getWithStorage().value(options))
+ .addBenchmarks(new FullTextSearcher(),
+ new NodeTypeSearcher(),
+ new FormatSearcher(),
+ new FacetSearcher(),
+ new LastModifiedSearcher(Date.LAST_2_HRS),
+ new LastModifiedSearcher(Date.LAST_24_HRS),
+ new LastModifiedSearcher(Date.LAST_7_DAYS),
+ new LastModifiedSearcher(Date.LAST_MONTH),
+ new LastModifiedSearcher(Date.LAST_YEAR),
+ new OrderByDate()),
+ new ScalabilityNodeSuite(scalabilityOptions.getWithStorage().value(options))
+ .addBenchmarks(new OrderBySearcher(),
+ new SplitOrderBySearcher(),
+ new OrderByOffsetPageSearcher(),
+ new SplitOrderByOffsetPageSearcher(),
+ new OrderByKeysetPageSearcher(),
+ new SplitOrderByKeysetPageSearcher(),
+ new MultiFilterOrderBySearcher(),
+ new MultiFilterSplitOrderBySearcher(),
+ new MultiFilterOrderByOffsetPageSearcher(),
+ new MultiFilterSplitOrderByOffsetPageSearcher(),
+ new MultiFilterOrderByKeysetPageSearcher(),
+ new MultiFilterSplitOrderByKeysetPageSearcher(),
+ new ConcurrentReader(),
+ new ConcurrentWriter()),
+ new ScalabilityNodeRelationshipSuite(scalabilityOptions.getWithStorage().value(options))
+ .addBenchmarks(new AggregateNodeSearcher())
+ )
+ );
+
+ }
+
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/AggregateNodeSearcher.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/AggregateNodeSearcher.java?rev=1875760&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/AggregateNodeSearcher.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/scalability/benchmarks/search/AggregateNodeSearcher.java Fri Mar 27 07:44:43 2020
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.scalability.benchmarks.search;
+
+import com.google.common.collect.Lists;
+
+import org.apache.jackrabbit.api.security.user.Authorizable;
+import org.jetbrains.annotations.NotNull;
+
+import javax.jcr.*;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+
+import java.util.List;
+import java.util.Random;
+
+import org.apache.jackrabbit.oak.scalability.suites.ScalabilityNodeRelationshipSuite;
+import org.apache.jackrabbit.oak.scalability.suites.ScalabilityAbstractSuite.ExecutionContext;
+
+/**
+ * Retrieves search property by iterating over nodes and then executes search using the retrieved
+ * criteria.
+ */
+public class AggregateNodeSearcher extends SearchScalabilityBenchmark {
+ private static final String RELATIONSHIPS = "relationships";
+
+ /**
+ * Queries for nodes with property satisfying a set of properties and ordering by the latest.
+ *
+ * @param qm the query manager
+ * @param context the execution context
+ * @return the query object
+ * @throws RepositoryException
+ */
+ protected Query getQuery(@NotNull QueryManager qm,
+ ExecutionContext context) throws RepositoryException {
+ List<String> relationships = (List<String>) context.getMap().get(RELATIONSHIPS);
+ // /jcr:root//element(*, ActivityType)[((id = 1234 or id = '1354'))] order by jcr:created
+ // descending
+ StringBuilder statement = new StringBuilder("");
+ statement.append("/jcr:root")
+ .append("//element(*, ")
+ .append(
+ (String) context.getMap().get(ScalabilityNodeRelationshipSuite.CTX_ACT_NODE_TYPE_PROP))
+ .append(")");
+ statement.append("[((");
+
+ // adding all the possible mime-types in an OR fashion
+ for (String relationship : relationships) {
+ statement.append(ScalabilityNodeRelationshipSuite.SOURCE_ID).append(" = '")
+ .append(relationship).append("' or ");
+ }
+
+ // removing latest ' or '
+ statement.delete(statement.lastIndexOf(" or "), statement.length());
+
+ statement.append("))]");
+ // order by jcr:created descending
+ statement.append(" order by").append(" @").append(ScalabilityNodeRelationshipSuite.CREATED)
+ .append(" descending");
+
+ LOG.debug("{}", statement);
+
+ return qm.createQuery(statement.toString(), Query.XPATH);
+ }
+
+ @Override
+ public void execute(Repository repository, Credentials credentials,
+ ExecutionContext context) throws Exception {
+ Session session = repository.login(credentials);
+ QueryManager qm;
+ try {
+ List<Authorizable> users = (List<Authorizable>) context.getMap()
+ .get(ScalabilityNodeRelationshipSuite.CTX_USER);
+ Random rand = new Random(99);
+ Authorizable user = users.get(rand.nextInt(users.size()));
+ List<String> targets = getRelatedUsers(session, user);
+ context.getMap().put(RELATIONSHIPS, targets);
+ qm = session.getWorkspace().getQueryManager();
+ search(qm, context);
+ context.getMap().remove(RELATIONSHIPS);
+ } catch (RepositoryException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private List<String> getRelatedUsers(Session session, Authorizable user)
+ throws RepositoryException {
+ List<String> targets = Lists.newArrayList();
+ Node relRootNode =
+ session.getNode(user.getPath() + "/" + ScalabilityNodeRelationshipSuite.RELATIONSHIPS);
+ NodeIterator children = relRootNode.getNodes();
+ while (children.hasNext()) {
+ Node node = children.nextNode();
+ targets.add(node.getProperty(ScalabilityNodeRelationshipSuite.TARGET_ID).getString());
+ }
+ return targets;
+ }
+}