You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ng...@apache.org on 2020/04/02 16:48:43 UTC
svn commit: r1876052 [1/2] - in /jackrabbit/oak/trunk: ./
oak-benchmarks-elastic/ oak-benchmarks-elastic/src/
oak-benchmarks-elastic/src/main/ oak-benchmarks-elastic/src/main/java/
oak-benchmarks-elastic/src/main/java/org/ oak-benchmarks-elastic/src/ma...
Author: ngupta
Date: Thu Apr 2 16:48:43 2020
New Revision: 1876052
URL: http://svn.apache.org/viewvc?rev=1876052&view=rev
Log:
OAK-8956 | Query benchmarks for search and indexing (additions for elastic and refactoring for lucene and solr)
Added:
jackrabbit/oak/trunk/oak-benchmarks-elastic/
jackrabbit/oak/trunk/oak-benchmarks-elastic/README.md
jackrabbit/oak/trunk/oak-benchmarks-elastic/pom.xml
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticBenchmarkRunner.java
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticFullTextNotGlobalSearchTest.java
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticFullTextSearchTest.java
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyFullTextSeparated.java
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyFullTextTest.java
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyTextSearchTest.java
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/util/
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/util/ElasticGlobalInitializer.java
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/run/
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkElasticCommand.java
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/run/ElasticMain.java
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/resources/
jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/resources/logback-benchmark.xml
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextNotGlobalSearchTest.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertySearchTest.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/ScalabilityLuceneCommand.java
jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/benchmark/SolrBenchmarkRunner.java
jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/run/
jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkSolrCommand.java
jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/run/SolrMain.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/PropertyFullTextTest.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/SearchTest.java
Removed:
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneScalabilityCommand.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSearchTest.java
Modified:
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneBenchmarkRunner.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextSearchTest.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFTSeparated.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFullTextTest.java
jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneMain.java
jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSolrSearchTest.java
jackrabbit/oak/trunk/oak-benchmarks/pom.xml
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AbstractTest.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkOptions.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/wikipedia/WikipediaImport.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnection.java
jackrabbit/oak/trunk/pom.xml
Added: jackrabbit/oak/trunk/oak-benchmarks-elastic/README.md
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-elastic/README.md?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-elastic/README.md (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-elastic/README.md Thu Apr 2 16:48:43 2020
@@ -0,0 +1,69 @@
+Oak Elastic Benchmark Jar
+=========================
+
+This jar is runnable and contains test related run modes.
+
+The following runmodes are currently available:
+
+ * benchmark : Run benchmark tests against different Oak repository fixtures.
+
+See the subsections below for more details on how to use these modes.
+
+Benchmark mode
+--------------
+
+The benchmark mode is used for executing various micro-benchmarks. It can
+be invoked like this:
+
+ $ java -jar oak-benchmarks-*.jar benchmark [options] [testcases] [fixtures]
+
+The following benchmark options are required :
+
+ --elasticHost - Elastic host server (e.g. localhost)
+ --elasticPort - Elastic server port (e.g 9200)
+ --elasticScheme - Eastic server scheme (e.g. http)
+
+
+Example Command for benchmark execution
+---------------------------------------
+
+The below command executes ElasticPropertyFullTextSeparated
+
+`benchmark ElasticPropertyFullTextSeparated Oak-Segment-Tar --wikipedia Path_to_wiki_dump_xml --elasticHost localhost --elasticPort 9200 --elasticScheme http`
+
+Available benchmarks are listed in [ElasticBenchmarkRunner](src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticBenchmarkRunner.java)
+
+Some other useful JVM parameters are -
+
+`-Dlogback.configurationFile=<path to logback-benchmark>\logback-benchmark.xml` (Useful for additional logging, [Sample](src/main/resources/logback-benchmark.xml))
+`-Druntime=180`(Change the benchmark execution time, default is 60 seconds)
+`-DskipWarmup=true`(skip warmup test execution)
+
+
+To add new benchmarks or to know about other options supported, please refer the README from oak-benchmarks at [0]
+
+
+[[0]Oak-Benchmarks README](../oak-benchmarks/README.md)
+
+
+License
+-------
+
+(see the top-level [LICENSE.txt](../LICENSE.txt) for full license details)
+
+Collective work: Copyright 2012 The Apache Software Foundation.
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
Added: jackrabbit/oak/trunk/oak-benchmarks-elastic/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-elastic/pom.xml?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-elastic/pom.xml (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-elastic/pom.xml Thu Apr 2 16:48:43 2020
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-parent</artifactId>
+ <version>1.27-SNAPSHOT</version>
+ <relativePath>../oak-parent/pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>oak-benchmarks-elastic</artifactId>
+
+ <properties>
+ <skip.deployment>true</skip.deployment>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-benchmarks</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-search-elastic</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jetbrains</groupId>
+ <artifactId>annotations</artifactId>
+ </dependency>
+
+ </dependencies>
+
+
+</project>
\ No newline at end of file
Added: jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticBenchmarkRunner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticBenchmarkRunner.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticBenchmarkRunner.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticBenchmarkRunner.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+
+import java.util.Arrays;
+
+public class ElasticBenchmarkRunner extends BenchmarkRunner {
+
+ private static ElasticsearchConnection coordinate;
+
+ public static void main(String[] args) throws Exception {
+ initOptionSet(args);
+ statsProvider = options.has(benchmarkOptions.getMetrics()) ? getStatsProvider() : StatisticsProvider.NOOP;
+ // Create an Elastic Client Connection here and pass down to all the tests
+ // And close the connection in this class itself
+ // Can't rely on the tear down methods of the downstream Benchmark classes because they don't handle the case
+ // where Exception could be thrown before the test execution is started - in that case if the connection is not closed here
+ // we have orphaned HttpClient's I/O disp threads that don't let the process exit.
+
+ try {
+ coordinate = new ElasticsearchConnection(benchmarkOptions.getElasticScheme().value(options),
+ benchmarkOptions.getElasticHost().value(options), benchmarkOptions.getElasticPort().value(options));
+
+ BenchmarkRunner.addToBenchMarkList(
+ Arrays.asList(
+ new ElasticFullTextSearchTest(benchmarkOptions.getWikipedia().value(options),
+ benchmarkOptions.getFlatStructure().value(options),
+ benchmarkOptions.getReport().value(options),
+ benchmarkOptions.getWithStorage().value(options),
+ coordinate),
+ new ElasticPropertyFullTextTest(benchmarkOptions.getWikipedia().value(options),
+ benchmarkOptions.getFlatStructure().value(options),
+ benchmarkOptions.getReport().value(options),
+ benchmarkOptions.getWithStorage().value(options),
+ coordinate),
+ new ElasticPropertyFullTextSeparated(benchmarkOptions.getWikipedia().value(options),
+ benchmarkOptions.getFlatStructure().value(options),
+ benchmarkOptions.getReport().value(options),
+ benchmarkOptions.getWithStorage().value(options),
+ coordinate),
+ new ElasticFullTextNotGlobalSearchTest(benchmarkOptions.getWikipedia().value(options),
+ benchmarkOptions.getFlatStructure().value(options),
+ benchmarkOptions.getReport().value(options),
+ benchmarkOptions.getWithStorage().value(options),
+ coordinate),
+ new ElasticPropertyTextSearchTest(benchmarkOptions.getWikipedia().value(options),
+ benchmarkOptions.getFlatStructure().value(options),
+ benchmarkOptions.getReport().value(options),
+ benchmarkOptions.getWithStorage().value(options),
+ coordinate)
+ )
+ );
+ BenchmarkRunner.main(args);
+ } finally {
+ coordinate.close();
+ }
+
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticFullTextNotGlobalSearchTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticFullTextNotGlobalSearchTest.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticFullTextNotGlobalSearchTest.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticFullTextNotGlobalSearchTest.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.fixture.JcrCreator;
+import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
+import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.index.ElasticsearchIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
+
+import javax.jcr.Repository;
+import java.io.File;
+
+import static com.google.common.collect.ImmutableSet.of;
+
+public class ElasticFullTextNotGlobalSearchTest extends SearchTest {
+
+ private ElasticsearchConnection coordinate;
+
+ public ElasticFullTextNotGlobalSearchTest(File dump, boolean flat, boolean doReport, Boolean storageEnabled, ElasticsearchConnection coordinate) {
+ super(dump, flat, doReport, storageEnabled);
+ this.coordinate = coordinate;
+ }
+
+ @Override
+ protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
+ if (fixture instanceof OakRepositoryFixture) {
+ return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() {
+ @Override
+ public Jcr customize(Oak oak) {
+ ElasticsearchIndexEditorProvider editorProvider = new ElasticsearchIndexEditorProvider(coordinate,
+ new ExtractedTextCache(10 * FileUtils.ONE_MB, 100));
+ ElasticsearchIndexProvider indexProvider = new ElasticsearchIndexProvider(coordinate);
+ oak.with(editorProvider)
+ .with(indexProvider)
+ .with(new PropertyIndexEditorProvider())
+ .with(new NodeTypeIndexProvider())
+ .with(new PropertyFullTextTest.FullTextPropertyInitialiser("elasticText", of("text"),
+ ElasticsearchIndexConstants.TYPE_ELASTICSEARCH).nodeScope().analyzed());
+ return new Jcr(oak);
+ }
+ });
+ }
+ return super.createRepository(fixture);
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticFullTextSearchTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticFullTextSearchTest.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticFullTextSearchTest.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticFullTextSearchTest.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.benchmark.util.ElasticGlobalInitializer;
+import org.apache.jackrabbit.oak.fixture.JcrCreator;
+import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
+import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.index.ElasticsearchIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
+
+import javax.jcr.Repository;
+import java.io.File;
+
+public class ElasticFullTextSearchTest extends SearchTest {
+
+ private ElasticsearchConnection coordinate;
+ private final String ELASTIC_GLOBAL_INDEX = "elasticGlobal";
+
+ public ElasticFullTextSearchTest(File dump, boolean flat, boolean doReport, Boolean storageEnabled, ElasticsearchConnection coordinate) {
+ super(dump, flat, doReport, storageEnabled);
+ this.coordinate = coordinate;
+ }
+
+ @Override
+ protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
+ if (fixture instanceof OakRepositoryFixture) {
+ return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() {
+ @Override
+ public Jcr customize(Oak oak) {
+ ElasticsearchIndexEditorProvider editorProvider = new ElasticsearchIndexEditorProvider(coordinate,
+ new ExtractedTextCache(10 * FileUtils.ONE_MB, 100));
+ ElasticsearchIndexProvider indexProvider = new ElasticsearchIndexProvider(coordinate);
+ oak.with(editorProvider)
+ .with(indexProvider)
+ .with(new PropertyIndexEditorProvider())
+ .with(new NodeTypeIndexProvider())
+ .with(new ElasticGlobalInitializer(ELASTIC_GLOBAL_INDEX, storageEnabled));
+ return new Jcr(oak);
+ }
+ });
+ }
+ return super.createRepository(fixture);
+ }
+
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyFullTextSeparated.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyFullTextSeparated.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyFullTextSeparated.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyFullTextSeparated.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.benchmark.util.ElasticGlobalInitializer;
+import org.apache.jackrabbit.oak.fixture.JcrCreator;
+import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
+import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.index.ElasticsearchIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.jcr.Repository;
+import java.io.File;
+
+import static com.google.common.collect.ImmutableSet.of;
+
+/**
+ * same as {@link ElasticPropertyFullTextTest} but will initialise a repository where the global
+ * full-text runs on a separate thread from elastic property.
+ */
+public class ElasticPropertyFullTextSeparated extends PropertyFullTextTest {
+
+ private String currentFixtureName;
+ private ElasticsearchConnection coordinate;
+ private final String ELASTIC_GLOBAL_INDEX = "elasticGlobal";
+
+ public ElasticPropertyFullTextSeparated(final File dump,
+ final boolean flat,
+ final boolean doReport,
+ final Boolean storageEnabled, ElasticsearchConnection coordinate) {
+ super(dump, flat, doReport, storageEnabled);
+ this.coordinate = coordinate;
+ }
+
+ @Override
+ public String getCurrentFixtureName() {
+ return currentFixtureName;
+ }
+
+ @Override
+ public String getCurrentTest() {
+ return this.getClass().getSimpleName();
+ }
+
+ @Override
+ protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
+ if (fixture instanceof OakRepositoryFixture) {
+ currentFixtureName = fixture.toString();
+ return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() {
+ @Override
+ public Jcr customize(Oak oak) {
+ ElasticsearchIndexEditorProvider editorProvider = new ElasticsearchIndexEditorProvider(coordinate,
+ new ExtractedTextCache(10 * FileUtils.ONE_MB, 100));
+ ElasticsearchIndexProvider indexProvider = new ElasticsearchIndexProvider(coordinate);
+ oak.with(editorProvider)
+ .with(indexProvider)
+ .with((new ElasticGlobalInitializer(ELASTIC_GLOBAL_INDEX, storageEnabled)).async("fulltext-async"))
+ // the WikipediaImporter set a property `title`
+ .with(new FullTextPropertyInitialiser("elasticTitle", of("title"),
+ ElasticsearchIndexConstants.TYPE_ELASTICSEARCH).async())
+ .withAsyncIndexing("async", 5)
+ .withAsyncIndexing("fulltext-async", 5);
+ return new Jcr(oak);
+ }
+ });
+ }
+ return super.createRepository(fixture);
+ }
+
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyFullTextTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyFullTextTest.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyFullTextTest.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyFullTextTest.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.benchmark.util.ElasticGlobalInitializer;
+import org.apache.jackrabbit.oak.benchmark.wikipedia.WikipediaImport;
+import org.apache.jackrabbit.oak.fixture.JcrCreator;
+import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
+import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.index.ElasticsearchIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.jcr.Repository;
+import java.io.File;
+
+import static com.google.common.collect.ImmutableSet.of;
+
+/**
+ * <p>
+ * Perform a benchmark on how long it takes for an ingested item to be available in a Elastic
+ * Property index when indexed in conjunction with a Global full-text Elastic (same thread). It makes
+ * use of the {@link WikipediaImport} to use a Wikipedia dump for content injestion.
+ * <p>
+ * Extend this class in lucene and elastic benchmarks and override the createRepository method to include respective
+ * Index Editor providers.
+ * </p>
+ * <p>
+ * Suggested dump:
+ * <a href="https://dumps.wikimedia.org/enwiki/20150403/enwiki-20150403-pages-articles.xml.bz2">https://dumps.wikimedia.org/enwiki/20150403/enwiki-20150403-pages-articles.xml.bz2</a>
+ * </p>
+ * <p>
+ * Usage example:
+ * </p>
+ * <p>
+ * <pre>
+ * java -Druntime=900 -Dlogback.configurationFile=logback-benchmark.xml \
+ * -jar ~/.m2/repository/org/apache/jackrabbit/oak-run/1.4-SNAPSHOT/oak-run-1.4-SNAPSHOT.jar \
+ * benchmark --wikipedia enwiki-20150403-pages-articles.xml.bz2 \
+ * --base ~/tmp/oak/ <Test Extending this class></> Oak-Tar Oak-Mongo
+ * </pre>
+ * <p>
+ * it will run the benchmark for 15 minutes against TarNS and MongoNS.
+ * </p>
+ */
+public class ElasticPropertyFullTextTest extends PropertyFullTextTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ElasticPropertyFullTextTest.class);
+ private String currentFixtureName;
+ private ElasticsearchConnection coordinate;
+ private final String ELASTIC_GLOBAL_INDEX = "elasticGlobal";
+
+ @Override
+ public String getCurrentFixtureName() {
+ return currentFixtureName;
+ }
+
+ @Override
+ public String getCurrentTest() {
+ return this.getClass().getSimpleName();
+ }
+
+
+ public ElasticPropertyFullTextTest(final File dump,
+ final boolean flat,
+ final boolean doReport,
+ final Boolean storageEnabled, ElasticsearchConnection coordinate) {
+ super(dump, flat, doReport, storageEnabled);
+ this.coordinate = coordinate;
+ }
+
+ @Override
+ protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
+ if (fixture instanceof OakRepositoryFixture) {
+ currentFixtureName = fixture.toString();
+ return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() {
+ @Override
+ public Jcr customize(Oak oak) {
+ ElasticsearchIndexEditorProvider editorProvider = new ElasticsearchIndexEditorProvider(coordinate,
+ new ExtractedTextCache(10 * FileUtils.ONE_MB, 100));
+ ElasticsearchIndexProvider indexProvider = new ElasticsearchIndexProvider(coordinate);
+ oak.with(editorProvider)
+ .with(indexProvider)
+ .with((new ElasticGlobalInitializer(ELASTIC_GLOBAL_INDEX, storageEnabled)).async())
+ // the WikipediaImporter set a property `title`
+ .with(new FullTextPropertyInitialiser("elasticTitle", of("title"),
+ ElasticsearchIndexConstants.TYPE_ELASTICSEARCH).async())
+ .withAsyncIndexing("async", 5);
+ return new Jcr(oak);
+ }
+ });
+ }
+ return super.createRepository(fixture);
+ }
+
+
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyTextSearchTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyTextSearchTest.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyTextSearchTest.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/ElasticPropertyTextSearchTest.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.fixture.JcrCreator;
+import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
+import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.index.ElasticsearchIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
+
+import javax.jcr.Repository;
+import javax.jcr.query.Query;
+import java.io.File;
+
+import static com.google.common.collect.ImmutableSet.of;
+
+public class ElasticPropertyTextSearchTest extends SearchTest {
+
+ private ElasticsearchConnection coordinate;
+
+ public ElasticPropertyTextSearchTest(File dump, boolean flat, boolean doReport, Boolean storageEnabled, ElasticsearchConnection coordinate) {
+ super(dump, flat, doReport, storageEnabled);
+ this.coordinate = coordinate;
+ }
+
+ @Override
+ protected String getQuery(String word) {
+ return "SELECT * FROM [nt:base] WHERE [text] = '" + word + "'";
+ }
+
+ @Override
+ protected String queryType() {
+ return Query.JCR_SQL2;
+ }
+
+ @Override
+ protected boolean isFullTextSearch() {
+ return false;
+ }
+
+ @Override
+ protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
+ if (fixture instanceof OakRepositoryFixture) {
+ return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() {
+ @Override
+ public Jcr customize(Oak oak) {
+ ElasticsearchIndexEditorProvider editorProvider = new ElasticsearchIndexEditorProvider(coordinate,
+ new ExtractedTextCache(10 * FileUtils.ONE_MB, 100));
+ ElasticsearchIndexProvider indexProvider = new ElasticsearchIndexProvider(coordinate);
+ oak.with(editorProvider)
+ .with(indexProvider)
+ .with(new PropertyIndexEditorProvider())
+ .with(new NodeTypeIndexProvider())
+ .with(new PropertyFullTextTest.FullTextPropertyInitialiser("elasticText", of("text"),
+ ElasticsearchIndexConstants.TYPE_ELASTICSEARCH));
+ return new Jcr(oak);
+ }
+ });
+ }
+ return super.createRepository(fixture);
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/util/ElasticGlobalInitializer.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/util/ElasticGlobalInitializer.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/util/ElasticGlobalInitializer.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/benchmark/util/ElasticGlobalInitializer.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark.util;
+
+
+import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.search.util.IndexHelper;
+import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Set;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
+
+
+public class ElasticGlobalInitializer implements RepositoryInitializer {
+
+ private final String name;
+
+ private final Set<String> propertyTypes;
+
+ private final Set<String> excludes;
+
+ private final String filePath;
+
+ private String async = null;
+
+ private Boolean storageEnabled;
+
+
+ public ElasticGlobalInitializer(String name) {
+ this(name, IndexHelper.JR_PROPERTY_INCLUDES, null, null, null);
+ }
+
+ public ElasticGlobalInitializer(String name, Boolean storageEnabled) {
+ this(name, IndexHelper.JR_PROPERTY_INCLUDES, null, null, storageEnabled);
+ }
+
+ public ElasticGlobalInitializer(String name, Set<String> propertyTypes) {
+ this(name, propertyTypes, null, null, null);
+ }
+
+ public ElasticGlobalInitializer(String name, Set<String> propertyTypes,
+ Set<String> excludes) {
+ this(name, propertyTypes, excludes, null, null);
+ }
+
+ public ElasticGlobalInitializer(String name, Set<String> propertyTypes,
+ String filePath) {
+ this(name, propertyTypes, null, filePath, null);
+ }
+
+ public ElasticGlobalInitializer(String name, Set<String> propertyTypes,
+ Set<String> excludes, String filePath, Boolean storageEnabled) {
+ this.name = name;
+ this.propertyTypes = propertyTypes;
+ this.excludes = excludes;
+ this.filePath = filePath;
+ this.storageEnabled = storageEnabled;
+ }
+
+ /**
+ * set the {@code async} property to "async".
+ *
+ * @return
+ */
+ public ElasticGlobalInitializer async() {
+ return async("async");
+ }
+
+ /**
+ * will set the {@code async} property to the provided value
+ *
+ * @param async
+ * @return
+ */
+ public ElasticGlobalInitializer async(@NotNull final String async) {
+ this.async = checkNotNull(async);
+ return this;
+ }
+
+ @Override
+ public void initialize(@NotNull NodeBuilder builder) {
+ if (builder.hasChildNode(INDEX_DEFINITIONS_NAME)
+ && builder.getChildNode(INDEX_DEFINITIONS_NAME).hasChildNode(this.name)) {
+ // do nothing
+ } else {
+ IndexHelper.newFTIndexDefinition(IndexUtils.getOrCreateOakIndex(builder),
+ this.name, ElasticsearchIndexConstants.TYPE_ELASTICSEARCH,
+ propertyTypes, excludes, async, storageEnabled);
+ }
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkElasticCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkElasticCommand.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkElasticCommand.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkElasticCommand.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import org.apache.jackrabbit.oak.benchmark.ElasticBenchmarkRunner;
+import org.apache.jackrabbit.oak.run.commons.Command;
+
+
+public class BenchmarkElasticCommand implements Command {
+ @Override
+ public void execute(String... args) throws Exception {
+ ElasticBenchmarkRunner.main(args);
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/run/ElasticMain.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/run/ElasticMain.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/run/ElasticMain.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/java/org/apache/jackrabbit/oak/run/ElasticMain.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.run.commons.Modes;
+import org.apache.jackrabbit.oak.run.commons.Utils;
+
+import static java.util.Arrays.copyOfRange;
+
+public class ElasticMain {
+
+ private static final Modes MODES = new Modes(ImmutableMap.<String, Command>of(
+ "benchmark", new BenchmarkElasticCommand()
+ ));
+
+ private ElasticMain() {
+ // Prevent instantiation.
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ Utils.printProductInfo(
+ args,
+ Main.class.getResourceAsStream("/META-INF/maven/org.apache.jackrabbit/oak-benchmarks-elastic/pom.properties")
+ );
+
+ Command c = MODES.getCommand("benchmark");
+ if (args.length > 0) {
+ c = MODES.getCommand(args[0]);
+
+ if (c == null) {
+ c = MODES.getCommand("benchmark");
+ }
+
+ args = copyOfRange(args, 1, args.length);
+ }
+
+ c.execute(args);
+ }
+
+
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/resources/logback-benchmark.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/resources/logback-benchmark.xml?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/resources/logback-benchmark.xml (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-elastic/src/main/resources/logback-benchmark.xml Thu Apr 2 16:48:43 2020
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<configuration>
+
+ <appender name="STDERR" class="ch.qos.logback.core.ConsoleAppender">
+ <target>System.err</target>
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <target>System.out</target>
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${user.home}/tmp/logs/oak.log</file>
+
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <!-- daily rollover -->
+ <fileNamePattern>${user.home}/tmp/logs/oak.%d.log.gz</fileNamePattern>
+
+ <maxHistory>365</maxHistory>
+ </rollingPolicy>
+
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+ </encoder>
+
+ </appender>
+
+ <!-- Disable warnings caused by JCR-3671 -->
+ <logger name="org.apache.jackrabbit.core.config.ConfigurationErrorHandler" level="ERROR"/>
+
+ <!-- Display info messages from the benchmark suite -->
+ <logger name="org.apache.jackrabbit.oak.benchmark" level="INFO"/>
+ <logger name="org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate" level="DEBUG" />
+ <logger name="org.apache.jackrabbit.oak.plugins.index.IndexUpdate" level="DEBUG" />
+
+ <root level="error">
+ <!-- <appender-ref ref="STDOUT" /> -->
+ <appender-ref ref="FILE" />
+ </root>
+
+</configuration>
Modified: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneBenchmarkRunner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneBenchmarkRunner.java?rev=1876052&r1=1876051&r2=1876052&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneBenchmarkRunner.java (original)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneBenchmarkRunner.java Thu Apr 2 16:48:43 2020
@@ -19,14 +19,13 @@ package org.apache.jackrabbit.oak.benchm
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
-import java.util.ArrayList;
import java.util.Arrays;
public class LuceneBenchmarkRunner extends BenchmarkRunner {
public static void main(String[] args) throws Exception {
- statsProvider = options.has(benchmarkOptions.getMetrics()) ? getStatsProvider() : StatisticsProvider.NOOP;
initOptionSet(args);
+ statsProvider = options.has(benchmarkOptions.getMetrics()) ? getStatsProvider() : StatisticsProvider.NOOP;
BenchmarkRunner.addToBenchMarkList(
Arrays.asList(
new LuceneFullTextSearchTest(
@@ -42,7 +41,15 @@ public class LuceneBenchmarkRunner exten
benchmarkOptions.getWikipedia().value(options),
benchmarkOptions.getFlatStructure().value(options),
benchmarkOptions.getReport().value(options), benchmarkOptions.getWithStorage().value(options)),
- new HybridIndexTest(benchmarkOptions.getBase().value(options), statsProvider)
+ new HybridIndexTest(benchmarkOptions.getBase().value(options), statsProvider),
+ new LuceneFullTextNotGlobalSearchTest(benchmarkOptions.getWikipedia().value(options),
+ benchmarkOptions.getFlatStructure().value(options),
+ benchmarkOptions.getReport().value(options),
+ benchmarkOptions.getWithStorage().value(options)),
+ new LucenePropertySearchTest(benchmarkOptions.getWikipedia().value(options),
+ benchmarkOptions.getFlatStructure().value(options),
+ benchmarkOptions.getReport().value(options),
+ benchmarkOptions.getWithStorage().value(options))
)
);
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextNotGlobalSearchTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextNotGlobalSearchTest.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextNotGlobalSearchTest.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextNotGlobalSearchTest.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.fixture.JcrCreator;
+import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
+import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+
+import javax.jcr.Repository;
+import java.io.File;
+import java.io.IOException;
+
+import static com.google.common.collect.ImmutableSet.of;
+
+/*
+Similar to {@Link LuceneFullTextSearchTest}. The only diff being this doesn't configure a global full text index
+ */
+public class LuceneFullTextNotGlobalSearchTest extends SearchTest {
+
+ private final boolean disableCopyOnRead = Boolean.getBoolean("disableCopyOnRead");
+
+ public LuceneFullTextNotGlobalSearchTest(File dump, boolean flat, boolean doReport, Boolean storageEnabled) {
+ super(dump, flat, doReport, storageEnabled);
+ }
+
+ @Override
+ protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
+ if (fixture instanceof OakRepositoryFixture) {
+ return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() {
+ @Override
+ public Jcr customize(Oak oak) {
+ LuceneIndexProvider provider = createLuceneIndexProvider();
+ oak.with((QueryIndexProvider) provider)
+ .with((Observer) provider)
+ .with(new LuceneIndexEditorProvider())
+ .with(new PropertyFullTextTest.FullTextPropertyInitialiser("luceneText", of("text"),
+ LuceneIndexConstants.TYPE_LUCENE).nodeScope().analyzed());
+ return new Jcr(oak);
+ }
+ });
+ }
+ return super.createRepository(fixture);
+ }
+
+ private LuceneIndexProvider createLuceneIndexProvider() {
+ if (!disableCopyOnRead) {
+ try {
+ IndexCopier copier = new IndexCopier(executorService, indexCopierDir, true);
+ return new LuceneIndexProvider(copier);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return new LuceneIndexProvider();
+ }
+}
Modified: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextSearchTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextSearchTest.java?rev=1876052&r1=1876051&r2=1876052&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextSearchTest.java (original)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LuceneFullTextSearchTest.java Thu Apr 2 16:48:43 2020
@@ -35,7 +35,7 @@ import javax.jcr.Repository;
import java.io.File;
import java.io.IOException;
-public class LuceneFullTextSearchTest extends FullTextSearchTest {
+public class LuceneFullTextSearchTest extends SearchTest {
private final boolean disableCopyOnRead = Boolean.getBoolean("disableCopyOnRead");
Modified: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFTSeparated.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFTSeparated.java?rev=1876052&r1=1876051&r2=1876052&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFTSeparated.java (original)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFTSeparated.java Thu Apr 2 16:48:43 2020
@@ -27,6 +27,7 @@ import org.apache.jackrabbit.oak.fixture
import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneInitializerHelper;
@@ -37,10 +38,22 @@ import org.apache.jackrabbit.oak.spi.que
* same as {@link LucenePropertyFullTextTest} but will initialise a repository where the global
* full-text runs on a separate thread from lucene property.
*/
-public class LucenePropertyFTSeparated extends LucenePropertyFullTextTest {
+public class LucenePropertyFTSeparated extends PropertyFullTextTest {
- public LucenePropertyFTSeparated(final File dump,
- final boolean flat,
+ String currentFixtureName;
+
+ @Override
+ public String getCurrentFixtureName() {
+ return currentFixtureName;
+ }
+
+ @Override
+ public String getCurrentTest() {
+ return this.getClass().getSimpleName();
+ }
+
+ public LucenePropertyFTSeparated(final File dump,
+ final boolean flat,
final boolean doReport,
final Boolean storageEnabled) {
super(dump, flat, doReport, storageEnabled);
@@ -50,21 +63,20 @@ public class LucenePropertyFTSeparated e
@Override
protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
if (fixture instanceof OakRepositoryFixture) {
- currentFixture = fixture.toString();
+ currentFixtureName = fixture.toString();
return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() {
@Override
public Jcr customize(Oak oak) {
LuceneIndexProvider provider = new LuceneIndexProvider();
oak.with((QueryIndexProvider) provider)
- .with((Observer) provider)
- .with(new LuceneIndexEditorProvider())
- .with(
- (new LuceneInitializerHelper("luceneGlobal", storageEnabled))
- .async("async-slow"))
- // the WikipediaImporter set a property `title`
- .with(new LucenePropertyInitialiser("luceneTitle", of("title")))
- .withAsyncIndexing("async", 5)
- .withAsyncIndexing("async-slow", 5);
+ .with((Observer) provider)
+ .with(new LuceneIndexEditorProvider())
+ .with((new LuceneInitializerHelper("luceneGlobal", storageEnabled))
+ .async("fulltext-async"))
+ // the WikipediaImporter set a property `title`
+ .with(new FullTextPropertyInitialiser("luceneTitle", of("title"), LuceneIndexConstants.TYPE_LUCENE).async())
+ .withAsyncIndexing("async", 5)
+ .withAsyncIndexing("fulltext-async", 5);
return new Jcr(oak);
}
});
Modified: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFullTextTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFullTextTest.java?rev=1876052&r1=1876051&r2=1876052&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFullTextTest.java (original)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertyFullTextTest.java Thu Apr 2 16:48:43 2020
@@ -16,54 +16,21 @@
*/
package org.apache.jackrabbit.oak.benchmark;
-import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.ImmutableSet.of;
-import static org.apache.jackrabbit.oak.api.Type.BOOLEAN;
-import static org.apache.jackrabbit.oak.api.Type.LONG;
-import static org.apache.jackrabbit.oak.api.Type.NAME;
-import static org.apache.jackrabbit.oak.api.Type.STRING;
-import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.ASYNC_PROPERTY_NAME;
-import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
-import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
-import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME;
-import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
-import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE;
-import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.COMPAT_MODE;
-import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INDEX_RULES;
-import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NAME;
-import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NODE;
-import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_PROPERTY_INDEX;
-
import java.io.File;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-
import javax.jcr.Repository;
-import javax.jcr.RepositoryException;
-import javax.jcr.Session;
-import javax.jcr.ValueFactory;
-import javax.jcr.query.Query;
-import javax.jcr.query.QueryManager;
-import javax.jcr.query.RowIterator;
-
import org.apache.jackrabbit.oak.Oak;
-import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.benchmark.wikipedia.WikipediaImport;
-import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.fixture.JcrCreator;
import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneInitializerHelper;
-import org.apache.jackrabbit.oak.plugins.tree.factories.TreeFactory;
import org.apache.jackrabbit.oak.spi.commit.Observer;
-import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer;
import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
-import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
-import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,99 +58,32 @@ import org.slf4j.LoggerFactory;
* it will run the benchmark for 15 minutes against TarNS and MongoNS.
* </p>
*/
-public class LucenePropertyFullTextTest extends AbstractTest<LucenePropertyFullTextTest.TestContext> {
+public class LucenePropertyFullTextTest extends PropertyFullTextTest {
private static final Logger LOG = LoggerFactory.getLogger(LucenePropertyFullTextTest.class);
- private WikipediaImport importer;
- private Thread asyncImporter;
- private boolean benchmarkCompleted, importerCompleted;
- Boolean storageEnabled;
- String currentFixture, currentTest;
-
- /**
- * context used across the tests
- */
- class TestContext {
- final Session session = loginWriter();
- final String title;
-
- public TestContext(@NotNull final String title) {
- this.title = checkNotNull(title);
- }
+ private String currentFixtureName;
+
+ @Override
+ public String getCurrentFixtureName() {
+ return currentFixtureName;
}
- /**
- * helper class to initialise the Lucene Property index definition
- */
- static class LucenePropertyInitialiser implements RepositoryInitializer {
- private String name;
- private Set<String> properties;
-
- public LucenePropertyInitialiser(@NotNull final String name,
- @NotNull final Set<String> properties) {
- this.name = checkNotNull(name);
- this.properties = checkNotNull(properties);
- }
-
- private boolean isAlreadyThere(@NotNull final NodeBuilder root) {
- return checkNotNull(root).hasChildNode(INDEX_DEFINITIONS_NAME) &&
- root.getChildNode(INDEX_DEFINITIONS_NAME).hasChildNode(name);
- }
-
- @Override
- public void initialize(final NodeBuilder builder) {
- if (!isAlreadyThere(builder)) {
- Tree t = TreeFactory.createTree(builder.child(INDEX_DEFINITIONS_NAME));
- t = t.addChild(name);
- t.setProperty("jcr:primaryType", INDEX_DEFINITIONS_NODE_TYPE, NAME);
- t.setProperty(COMPAT_MODE, 2L, LONG);
- t.setProperty(TYPE_PROPERTY_NAME, TYPE_LUCENE, STRING);
- t.setProperty(ASYNC_PROPERTY_NAME, "async", STRING);
- t.setProperty(REINDEX_PROPERTY_NAME, true);
-
- t = t.addChild(INDEX_RULES);
- t.setOrderableChildren(true);
- t.setProperty("jcr:primaryType", "nt:unstructured", NAME);
-
- t = t.addChild("nt:base");
-
- Tree propnode = t.addChild(PROP_NODE);
- propnode.setOrderableChildren(true);
- propnode.setProperty("jcr:primaryType", "nt:unstructured", NAME);
-
- for (String p : properties) {
- Tree t1 = propnode.addChild(PathUtils.getName(p));
- t1.setProperty(PROP_PROPERTY_INDEX, true, BOOLEAN);
- t1.setProperty(PROP_NAME, p);
- }
- }
- }
+ @Override
+ public String getCurrentTest() {
+ return this.getClass().getSimpleName();
}
-
- /**
- * reference to the last added title. Used for looking up with queries.
- */
- private AtomicReference<String> lastTitle = new AtomicReference<String>();
-
- public LucenePropertyFullTextTest(final File dump,
- final boolean flat,
- final boolean doReport,
- final Boolean storageEnabled) {
- this.importer = new WikipediaImport(dump, flat, doReport) {
- @Override
- protected void pageAdded(String title, String text) {
- LOG.trace("Setting title: {}", title);
- lastTitle.set(title);
- }
- };
- this.storageEnabled = storageEnabled;
- this.currentTest = this.getClass().getSimpleName();
+
+ public LucenePropertyFullTextTest(final File dump,
+ final boolean flat,
+ final boolean doReport,
+ final Boolean storageEnabled) {
+ super(dump, flat, doReport, storageEnabled);
}
@Override
protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
if (fixture instanceof OakRepositoryFixture) {
- currentFixture = fixture.toString();
+ currentFixtureName = fixture.toString();
return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() {
@Override
public Jcr customize(Oak oak) {
@@ -193,7 +93,7 @@ public class LucenePropertyFullTextTest
.with(new LuceneIndexEditorProvider())
.with((new LuceneInitializerHelper("luceneGlobal", storageEnabled)).async())
// the WikipediaImporter set a property `title`
- .with(new LucenePropertyInitialiser("luceneTitle", of("title")))
+ .with(new FullTextPropertyInitialiser("luceneTitle", of("title"), LuceneIndexConstants.TYPE_LUCENE).async())
.withAsyncIndexing("async", 5);
return new Jcr(oak);
}
@@ -202,94 +102,4 @@ public class LucenePropertyFullTextTest
return super.createRepository(fixture);
}
- @Override
- protected void beforeSuite() throws Exception {
- LOG.debug("beforeSuite() - {} - {}", currentFixture, currentTest);
- benchmarkCompleted = false;
- importerCompleted = false;
- asyncImporter = new Thread(new Runnable() {
- @Override
- public void run() {
- try {
- importer.importWikipedia(loginWriter());
- } catch (Exception e) {
- LOG.error("Error while importing the dump. Trying to halt everything.", e);
- importerCompleted = true;
- } finally {
- if (!benchmarkCompleted) {
- importerCompleted = true;
- issueHaltRequest("Wikipedia import completed.");
- }
- }
- }
- });
- asyncImporter.start();
-
- // allowing the async index to catch up.
- TimeUnit.SECONDS.sleep(10);
- }
-
- @Override
- protected void afterSuite() throws Exception {
- LOG.debug("afterSuite() - {} - {}", currentFixture, currentTest);
- asyncImporter.join();
- }
-
- @Override
- protected void runTest() throws Exception {
- if (lastTitle.get() == null) {
- return;
- }
- runTest(new TestContext(lastTitle.get()));
- }
-
- @Override
- protected void runTest(final TestContext ec) throws Exception {
- if (importerCompleted) {
- return;
- }
- final long maxWait = TimeUnit.MINUTES.toMillis(5);
- final long waitUnit = 50;
- long sleptSoFar = 0;
-
- while (!performQuery(ec) && sleptSoFar < maxWait) {
- LOG.trace("title '{}' not found. Waiting and retry. sleptSoFar: {}ms", ec.title,
- sleptSoFar);
- sleptSoFar += waitUnit;
- TimeUnit.MILLISECONDS.sleep(waitUnit);
- }
-
- if (sleptSoFar < maxWait) {
- // means we exited the loop as we found it.
- LOG.info("{} - {} - title '{}' found with a wait/try of {}ms", currentFixture,
- currentTest, ec.title, sleptSoFar);
- } else {
- LOG.warn("{} - {} - title '{}' timed out with a way/try of {}ms.", currentFixture,
- currentTest, ec.title, sleptSoFar);
- }
- }
-
- private boolean performQuery(@NotNull final TestContext ec) throws RepositoryException {
- QueryManager qm = ec.session.getWorkspace().getQueryManager();
- ValueFactory vf = ec.session.getValueFactory();
- Query q = qm.createQuery("SELECT * FROM [nt:base] WHERE [title] = $title", Query.JCR_SQL2);
- q.bindValue("title", vf.createValue(ec.title));
- LOG.trace("statement: {} - title: {}", q.getStatement(), ec.title);
- RowIterator rows = q.execute().getRows();
- if (rows.hasNext()) {
- rows.nextRow().getPath();
- return true;
- } else {
- return false;
- }
- }
-
- @Override
- protected void issueHaltChildThreads() {
- if (!importerCompleted) {
- LOG.info("benchmark completed. Issuing an halt for the importer");
- benchmarkCompleted = true;
- this.importer.issueHaltImport();
- }
- }
}
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertySearchTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertySearchTest.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertySearchTest.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/benchmark/LucenePropertySearchTest.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.fixture.JcrCreator;
+import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
+import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+
+import javax.jcr.Repository;
+import javax.jcr.query.Query;
+import java.io.File;
+import java.io.IOException;
+
+import static com.google.common.collect.ImmutableSet.of;
+
+public class LucenePropertySearchTest extends SearchTest {
+
+ private final boolean disableCopyOnRead = Boolean.getBoolean("disableCopyOnRead");
+
+ public LucenePropertySearchTest(File dump, boolean flat, boolean doReport, Boolean storageEnabled) {
+ super(dump, flat, doReport, storageEnabled);
+ }
+
+ @Override
+ protected String getQuery(String word) {
+ return "SELECT * FROM [nt:base] WHERE [text] = '" + word + "'";
+ }
+
+ @Override
+ protected String queryType() {
+ return Query.JCR_SQL2;
+ }
+
+ @Override
+ protected boolean isFullTextSearch() {
+ return false;
+ }
+
+ @Override
+ protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
+ if (fixture instanceof OakRepositoryFixture) {
+ return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCreator() {
+ @Override
+ public Jcr customize(Oak oak) {
+ LuceneIndexProvider provider = createLuceneIndexProvider();
+ oak.with((QueryIndexProvider) provider)
+ .with((Observer) provider)
+ .with(new LuceneIndexEditorProvider())
+ .with(new PropertyFullTextTest.FullTextPropertyInitialiser("luceneText", of("text"),
+ LuceneIndexConstants.TYPE_LUCENE));
+ return new Jcr(oak);
+ }
+ });
+ }
+ return super.createRepository(fixture);
+ }
+
+ private LuceneIndexProvider createLuceneIndexProvider() {
+ if (!disableCopyOnRead) {
+ try {
+ IndexCopier copier = new IndexCopier(executorService, indexCopierDir, true);
+ return new LuceneIndexProvider(copier);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return new LuceneIndexProvider();
+ }
+}
Modified: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneMain.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneMain.java?rev=1876052&r1=1876051&r2=1876052&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneMain.java (original)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/LuceneMain.java Thu Apr 2 16:48:43 2020
@@ -28,7 +28,7 @@ public class LuceneMain {
private static final Modes MODES = new Modes(ImmutableMap.<String, Command>of(
"benchmark", new BenchmarkLuceneCommand(),
- "scalability", new ScalabilityCommand()
+ "scalability", new ScalabilityLuceneCommand()
));
private LuceneMain() {
Added: jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/ScalabilityLuceneCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/ScalabilityLuceneCommand.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/ScalabilityLuceneCommand.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-lucene/src/main/java/org/apache/jackrabbit/oak/run/ScalabilityLuceneCommand.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+
+import org.apache.jackrabbit.oak.scalability.LuceneScalabilityRunner;
+
+public class ScalabilityLuceneCommand extends ScalabilityCommand {
+
+ @Override
+ public void execute(String... args) throws Exception {
+ LuceneScalabilityRunner.main(args);
+ }
+}
Modified: jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSolrSearchTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSolrSearchTest.java?rev=1876052&r1=1876051&r2=1876052&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSolrSearchTest.java (original)
+++ jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSolrSearchTest.java Thu Apr 2 16:48:43 2020
@@ -48,7 +48,7 @@ import org.jetbrains.annotations.NotNull
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class FullTextSolrSearchTest extends FullTextSearchTest {
+public class FullTextSolrSearchTest extends SearchTest {
private final Logger log = LoggerFactory.getLogger(getClass());
Added: jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/benchmark/SolrBenchmarkRunner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/benchmark/SolrBenchmarkRunner.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/benchmark/SolrBenchmarkRunner.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/benchmark/SolrBenchmarkRunner.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+
+import java.util.Arrays;
+
+public class SolrBenchmarkRunner extends BenchmarkRunner {
+
+ public static void main(String[] args) throws Exception {
+ initOptionSet(args);
+ statsProvider = options.has(benchmarkOptions.getMetrics()) ? getStatsProvider() : StatisticsProvider.NOOP;
+
+ BenchmarkRunner.addToBenchMarkList(
+ Arrays.asList(new FullTextSolrSearchTest(
+ benchmarkOptions.getWikipedia().value(options),
+ benchmarkOptions.getFlatStructure().value(options),
+ benchmarkOptions.getReport().value(options), benchmarkOptions.getWithStorage().value(options),
+ benchmarkOptions.getWithServer().value(options)))
+ );
+
+ BenchmarkRunner.main(args);
+ }
+}
Added: jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkSolrCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkSolrCommand.java?rev=1876052&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkSolrCommand.java (added)
+++ jackrabbit/oak/trunk/oak-benchmarks-solr/src/main/java/org/apache/jackrabbit/oak/run/BenchmarkSolrCommand.java Thu Apr 2 16:48:43 2020
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import org.apache.jackrabbit.oak.benchmark.SolrBenchmarkRunner;
+import org.apache.jackrabbit.oak.run.commons.Command;
+
+public class BenchmarkSolrCommand implements Command {
+ @Override
+ public void execute(String... args) throws Exception {
+ SolrBenchmarkRunner.main(args);
+ }
+}