You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ng...@apache.org on 2020/03/26 11:02:42 UTC
svn commit: r1875701 [1/2] - in /jackrabbit/oak/trunk/oak-search-elastic: ./
src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/
src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/
src/main/java/org/apache/jackra...
Author: ngupta
Date: Thu Mar 26 11:02:41 2020
New Revision: 1875701
URL: http://svn.apache.org/viewvc?rev=1875701&view=rev
Log:
OAK-8965 | oak-search-elastic: review and update current code
Added:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnection.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexDescriptor.java
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnectionTest.java
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexProviderServiceTest.java
Removed:
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/DefaultElasticsearchIndexCoordinateFactory.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnectionFactory.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchCoordinate.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchCoordinateImpl.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexCoordinate.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexCoordinateFactory.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexCoordinateImpl.java
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchDockerRule.java
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchManagementRule.java
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchTestUtils.java
Modified:
jackrabbit/oak/trunk/oak-search-elastic/pom.xml
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexProviderService.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocument.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorContext.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorProvider.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriter.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriterFactory.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexNode.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexProvider.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexStatistics.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchSearcher.java
jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/util/TermQueryBuilderFactory.java
jackrabbit/oak/trunk/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchPropertyIndexTest.java
Modified: jackrabbit/oak/trunk/oak-search-elastic/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/pom.xml?rev=1875701&r1=1875700&r2=1875701&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-search-elastic/pom.xml Thu Mar 26 11:02:41 2020
@@ -33,8 +33,8 @@
<description>Oak Elasticsearch integration subproject</description>
<properties>
- <elasticsearch.version>7.1.1</elasticsearch.version>
- <lucene.version>8.0.0</lucene.version>
+ <elasticsearch.version>7.6.0</elasticsearch.version>
+ <lucene.version>8.4.0</lucene.version>
</properties>
<build>
@@ -44,13 +44,30 @@
<artifactId>maven-bundle-plugin</artifactId>
<configuration>
<instructions>
- <!-- TODO: of late not embedding this seems to working for locally, BUT it had failed initially. Need to verify-->
- <!-- We need to embed lucene as it's required by various QueryBuilders-->
+ <_exportcontents>
+ org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexProviderService
+ </_exportcontents>
+ <Import-Package>
+ !com.carrotsearch.randomizedtesting.*,
+ !com.google.common.geometry.*,
+ !com.sun.management.*,
+ !jdk.net.*,
+ !org.apache.avalon.framework.logger.*,
+ !org.apache.log.*,
+ !org.apache.logging.*,
+ !org.elasticsearch.geometry.*,
+ !org.joda.convert.*,
+ !org.locationtech.jts.geom.*,
+ !org.locationtech.spatial4j.*,
+ !sun.misc.*,
+ *
+ </Import-Package>
<Embed-Dependency>
- lucene-core;inline=true
- elasticsearch-rest-high-level-client;inline=true
+ oak-search;inline=true,
+ elasticsearch;groupId=org.elasticsearch,
+ *;scope=compile|runtime
</Embed-Dependency>
- <Export-Package>!*</Export-Package>
+ <Embed-Transitive>true</Embed-Transitive>
</instructions>
</configuration>
</plugin>
@@ -91,76 +108,91 @@
<scope>provided</scope>
</dependency>
+ <!-- Elastic/Lucene -->
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-high-level-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch-x-content</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${lucene.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-api</artifactId>
<version>${project.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-core-spi</artifactId>
<version>${project.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-store-spi</artifactId>
<version>${project.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-query-spi</artifactId>
<version>${project.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-core</artifactId>
<version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>oak-store-document</artifactId>
- <version>${project.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-search</artifactId>
<version>${project.version}</version>
+ <scope>provided</scope>
</dependency>
<!-- Logging -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
+ <scope>provided</scope>
</dependency>
<!-- Nullability annotations -->
<dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
+ <scope>provided</scope>
</dependency>
<!-- Test Dependencies -->
<dependency>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-classic</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>${elasticsearch.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>${lucene.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-core</artifactId>
<version>${project.version}</version>
@@ -169,132 +201,27 @@
</dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
- <artifactId>oak-blob-plugins</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>oak-store-spi</artifactId>
+ <artifactId>oak-security-spi</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>oak-segment-tar</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>oak-segment-tar</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>oak-store-document</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>oak-jcr</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>oak-jcr</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>oak-commons</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-jcr-tests</artifactId>
- <version>${jackrabbit.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>jackrabbit-core</artifactId>
- <version>${jackrabbit.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.tika</groupId>
- <artifactId>tika-parsers</artifactId>
- <version>${tika.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>org.apache.sling</groupId>
<artifactId>org.apache.sling.testing.osgi-mock</artifactId>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-all</artifactId>
- <version>1.3</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-exec</artifactId>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
<version>1.3</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>2.8.0</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.arakelian</groupId>
- <artifactId>docker-junit-rule</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.jackrabbit</groupId>
- <artifactId>oak-search</artifactId>
- <classifier>tests</classifier>
- <version>${project.version}</version>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>1.12.5</version>
<scope>test</scope>
</dependency>
-
</dependencies>
</project>
Added: jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnection.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnection.java?rev=1875701&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnection.java (added)
+++ jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchConnection.java Thu Mar 26 11:02:41 2020
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.elasticsearch;
+
+import org.apache.http.HttpHost;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestHighLevelClient;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Supplier;
+
+/**
+ * This class represents an Elasticsearch Connection with the related <code>RestHighLevelClient</code>.
+ * As per Elasticsearch documentation: the client is thread-safe, there should be one instance per application and it
+ * must be closed when it is not needed anymore.
+ * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/_changing_the_client_8217_s_initialization_code.html
+ *
+ * The getClient() initializes the rest client on the first call.
+ * Once close() is invoked this instance cannot be used anymore.
+ */
+public class ElasticsearchConnection implements Closeable {
+
+ protected static final String SCHEME_PROP = "elasticsearch.scheme";
+ protected static final String DEFAULT_SCHEME = "http";
+ protected static final String HOST_PROP = "elasticsearch.host";
+ protected static final String DEFAULT_HOST = "127.0.0.1";
+ protected static final String PORT_PROP = "elasticsearch.port";
+ protected static final int DEFAULT_PORT = 9200;
+
+ protected static final Supplier<ElasticsearchConnection> defaultConnection = () ->
+ new ElasticsearchConnection(DEFAULT_SCHEME, DEFAULT_HOST, DEFAULT_PORT);
+
+ private String scheme;
+ private String host;
+ private int port;
+
+ private volatile RestHighLevelClient client;
+
+ private AtomicBoolean isClosed = new AtomicBoolean(false);
+
+ protected ElasticsearchConnection(String scheme, String host, Integer port) {
+ if (scheme == null || host == null || port == null) {
+ throw new IllegalArgumentException();
+ }
+ this.scheme = scheme;
+ this.host = host;
+ this.port = port;
+ }
+
+ public RestHighLevelClient getClient() {
+ if (isClosed.get()) {
+ throw new IllegalStateException("Already closed");
+ }
+
+ // double checked locking to get good performance and avoid double initialization
+ if (client == null) {
+ synchronized (this) {
+ if (client == null) {
+ client = new RestHighLevelClient(RestClient.builder(new HttpHost(host, port, scheme)));
+ }
+ }
+ }
+ return client;
+ }
+
+ public String getScheme() {
+ return scheme;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (client != null) {
+ client.close();
+ }
+ isClosed.set(true);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ElasticsearchConnection that = (ElasticsearchConnection) o;
+ return port == that.port &&
+ Objects.equals(scheme, that.scheme) &&
+ Objects.equals(host, that.host);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getScheme(), getHost(), getPort());
+ }
+
+ @Override
+ public String toString() {
+ return getScheme() + "://" + getHost() + ":" + getPort();
+ }
+
+}
Added: jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexDescriptor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexDescriptor.java?rev=1875701&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexDescriptor.java (added)
+++ jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexDescriptor.java Thu Mar 26 11:02:41 2020
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.elasticsearch;
+
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Objects;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import static org.elasticsearch.common.Strings.INVALID_FILENAME_CHARS;
+
+public class ElasticsearchIndexDescriptor {
+
+ private static final int MAX_NAME_LENGTH = 255;
+
+ private static final String INVALID_CHARS_REGEX = Pattern.quote(INVALID_FILENAME_CHARS
+ .stream()
+ .map(Object::toString)
+ .collect(Collectors.joining("")));
+
+ private final ElasticsearchConnection connection;
+ private final String indexName;
+
+ public ElasticsearchIndexDescriptor(@NotNull ElasticsearchConnection connection, IndexDefinition indexDefinition) {
+ this.connection = connection;
+ this.indexName = getRemoteIndexName(indexDefinition);
+ }
+
+ public RestHighLevelClient getClient() {
+ return connection.getClient();
+ }
+
+ public String getIndexName() {
+ return indexName;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(connection, indexName);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ElasticsearchIndexDescriptor that = (ElasticsearchIndexDescriptor) o;
+ return connection.equals(that.connection) &&
+ indexName.equals(that.indexName);
+ }
+
+ private String getRemoteIndexName(IndexDefinition definition) {
+ String suffix = definition.getUniqueId();
+
+ if (suffix == null) {
+ suffix = String.valueOf(definition.getReindexCount());
+ }
+
+ return getESSafeIndexName(definition.getIndexPath() + "-" + suffix);
+ }
+
+ /**
+ * <ul>
+ * <li>abc -> abc</li>
+ * <li>xy:abc -> xyabc</li>
+ * <li>/oak:index/abc -> abc</li>
+ * </ul>
+ * <p>
+ * The resulting file name would be truncated to MAX_NAME_LENGTH
+ */
+ private static String getESSafeIndexName(String indexPath) {
+ String name = StreamSupport
+ .stream(PathUtils.elements(indexPath).spliterator(), false)
+ .limit(3) //Max 3 nodeNames including oak:index which is the immediate parent for any indexPath
+ .filter(p -> !"oak:index".equals(p))
+ .map(ElasticsearchIndexDescriptor::getESSafeName)
+ .collect(Collectors.joining("_"));
+
+ if (name.length() > MAX_NAME_LENGTH) {
+ name = name.substring(0, MAX_NAME_LENGTH);
+ }
+ return name;
+ }
+
+ /**
+ * Convert {@code e} to Elasticsearch safe index name.
+ * Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html
+ */
+ private static String getESSafeName(String suggestedIndexName) {
+ return suggestedIndexName.replaceAll(INVALID_CHARS_REGEX, "").toLowerCase();
+ }
+
+}
Modified: jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexProviderService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexProviderService.java?rev=1875701&r1=1875700&r2=1875701&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexProviderService.java (original)
+++ jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/ElasticsearchIndexProviderService.java Thu Mar 26 11:02:41 2020
@@ -16,11 +16,15 @@
*/
package org.apache.jackrabbit.oak.plugins.index.elasticsearch;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import org.apache.commons.io.FilenameUtils;
-import org.apache.felix.scr.annotations.*;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Property;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.ReferencePolicy;
+import org.apache.felix.scr.annotations.ReferencePolicyOption;
import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean;
import org.apache.jackrabbit.oak.cache.CacheStats;
import org.apache.jackrabbit.oak.commons.IOUtils;
@@ -31,22 +35,24 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.elasticsearch.query.ElasticsearchIndexProvider;
import org.apache.jackrabbit.oak.plugins.index.fulltext.PreExtractedTextProvider;
import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
-import org.apache.jackrabbit.oak.plugins.index.search.TextExtractionStatsMBean;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
import org.apache.jackrabbit.oak.spi.whiteboard.Registration;
import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.jetbrains.annotations.NotNull;
import org.osgi.framework.BundleContext;
import org.osgi.framework.ServiceRegistration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.util.ArrayList;
import java.util.Dictionary;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
-import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.commons.io.FileUtils.ONE_MB;
import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.registerMBean;
@@ -83,26 +89,23 @@ public class ElasticsearchIndexProviderS
)
private static final String PROP_PRE_EXTRACTED_TEXT_ALWAYS_USE = "alwaysUsePreExtractedCache";
- private static final String PROP_ELASTICSEARCH_SCHEME_DEFAULT = "http";
-// @Property(
-// value = PROP_ELASTICSEARCH_SCHEME_DEFAULT,
-// label = "Elasticsearch connection scheme"
-// )
- private static final String PROP_ELASTICSEARCH_SCHEME = "elasticsearch.scheme";
-
- private static final String PROP_ELASTICSEARCH_HOST_DEFAULT = "localhost";
-// @Property(
-// value = PROP_ELASTICSEARCH_HOST_DEFAULT,
-// label = "Elasticsearch connection host"
-// )
- private static final String PROP_ELASTICSEARCH_HOST = "elasticsearch.host";
-
- private static final int PROP_ELASTICSEARCH_PORT_DEFAULT = 9200;
-// @Property(
-// intValue = PROP_ELASTICSEARCH_PORT_DEFAULT,
-// label = "Elasticsearch connection port"
-// )
- private static final String PROP_ELASTICSEARCH_PORT = "elasticsearch.port";
+ @Property(
+ value = ElasticsearchConnection.DEFAULT_SCHEME,
+ label = "Elasticsearch connection scheme"
+ )
+ private static final String PROP_ELASTICSEARCH_SCHEME = ElasticsearchConnection.SCHEME_PROP;
+
+ @Property(
+ value = ElasticsearchConnection.DEFAULT_HOST,
+ label = "Elasticsearch connection host"
+ )
+ private static final String PROP_ELASTICSEARCH_HOST = ElasticsearchConnection.HOST_PROP;
+
+ @Property(
+ intValue = ElasticsearchConnection.DEFAULT_PORT,
+ label = "Elasticsearch connection port"
+ )
+ private static final String PROP_ELASTICSEARCH_PORT = ElasticsearchConnection.PORT_PROP;
@Property(
label = "Local text extraction cache path",
@@ -121,26 +124,25 @@ public class ElasticsearchIndexProviderS
private ExtractedTextCache extractedTextCache;
- private ElasticsearchConnectionFactory connectionFactory = null;
-
- private final List<ServiceRegistration> regs = Lists.newArrayList();
- private final List<Registration> oakRegs = Lists.newArrayList();
+ private final List<ServiceRegistration> regs = new ArrayList<>();
+ private final List<Registration> oakRegs = new ArrayList<>();
private Whiteboard whiteboard;
private File textExtractionDir;
+ private ElasticsearchConnection elasticsearchConnection;
+
@Activate
private void activate(BundleContext bundleContext, Map<String, ?> config) {
whiteboard = new OsgiWhiteboard(bundleContext);
- initializeTextExtractionDir(bundleContext, config);
- initializeExtractedTextCache(config, statisticsProvider);
+ //initializeTextExtractionDir(bundleContext, config);
+ //initializeExtractedTextCache(config, statisticsProvider);
- connectionFactory = new ElasticsearchConnectionFactory();
- ElasticsearchIndexCoordinateFactory esIndexCoordFactory = getElasticsearchIndexCoordinateFactory(config);
+ elasticsearchConnection = getElasticsearchCoordinate(config);
- registerIndexProvider(bundleContext, esIndexCoordFactory);
- registerIndexEditor(bundleContext, esIndexCoordFactory);
+ registerIndexProvider(bundleContext);
+ registerIndexEditor(bundleContext);
}
@Deactivate
@@ -153,33 +155,32 @@ public class ElasticsearchIndexProviderS
reg.unregister();
}
- IOUtils.closeQuietly(connectionFactory);
- connectionFactory = null;
+ IOUtils.closeQuietly(elasticsearchConnection);
if (extractedTextCache != null) {
extractedTextCache.close();
}
}
- private void registerIndexProvider(BundleContext bundleContext, ElasticsearchIndexCoordinateFactory esIndexCoordFactory) {
- ElasticsearchIndexProvider indexProvider = new ElasticsearchIndexProvider(esIndexCoordFactory);
+ private void registerIndexProvider(BundleContext bundleContext) {
+ ElasticsearchIndexProvider indexProvider = new ElasticsearchIndexProvider(elasticsearchConnection);
Dictionary<String, Object> props = new Hashtable<>();
props.put("type", ElasticsearchIndexConstants.TYPE_ELASTICSEARCH);
- regs.add(bundleContext.registerService(IndexEditorProvider.class.getName(), indexProvider, props));
+ regs.add(bundleContext.registerService(QueryIndexProvider.class.getName(), indexProvider, props));
}
- private void registerIndexEditor(BundleContext bundleContext, ElasticsearchIndexCoordinateFactory esIndexCoordFactory) {
- ElasticsearchIndexEditorProvider editorProvider = new ElasticsearchIndexEditorProvider(esIndexCoordFactory, extractedTextCache);
+ private void registerIndexEditor(BundleContext bundleContext) {
+ ElasticsearchIndexEditorProvider editorProvider = new ElasticsearchIndexEditorProvider(elasticsearchConnection, extractedTextCache);
Dictionary<String, Object> props = new Hashtable<>();
props.put("type", ElasticsearchIndexConstants.TYPE_ELASTICSEARCH);
regs.add(bundleContext.registerService(IndexEditorProvider.class.getName(), editorProvider, props));
- oakRegs.add(registerMBean(whiteboard,
- TextExtractionStatsMBean.class,
- editorProvider.getExtractedTextCache().getStatsMBean(),
- TextExtractionStatsMBean.TYPE,
- "TextExtraction statistics"));
+// oakRegs.add(registerMBean(whiteboard,
+// TextExtractionStatsMBean.class,
+// editorProvider.getExtractedTextCache().getStatsMBean(),
+// TextExtractionStatsMBean.TYPE,
+// "TextExtraction statistics"));
}
private void initializeExtractedTextCache(Map<String, ?> config, StatisticsProvider statisticsProvider) {
@@ -196,11 +197,11 @@ public class ElasticsearchIndexProviderS
alwaysUsePreExtractedCache,
textExtractionDir,
statisticsProvider);
- if (extractedTextProvider != null){
+ if (extractedTextProvider != null) {
registerExtractedTextProvider(extractedTextProvider);
}
CacheStats stats = extractedTextCache.getCacheStats();
- if (stats != null){
+ if (stats != null) {
oakRegs.add(registerMBean(whiteboard,
CacheStatsMBean.class, stats,
CacheStatsMBean.TYPE, stats.getName()));
@@ -209,28 +210,30 @@ public class ElasticsearchIndexProviderS
}
}
- void initializeTextExtractionDir(BundleContext bundleContext, Map<String, ?> config) {
+ private void initializeTextExtractionDir(BundleContext bundleContext, Map<String, ?> config) {
String textExtractionDir = PropertiesUtil.toString(config.get(PROP_LOCAL_TEXT_EXTRACTION_DIR), null);
- if (Strings.isNullOrEmpty(textExtractionDir)) {
+ if (textExtractionDir == null || textExtractionDir.trim().isEmpty()) {
String repoHome = bundleContext.getProperty(REPOSITORY_HOME);
- if (repoHome != null){
+ if (repoHome != null) {
textExtractionDir = FilenameUtils.concat(repoHome, "index");
}
}
- checkNotNull(textExtractionDir, "Text extraction directory cannot be determined as neither " +
- "directory path [%s] nor repository home [%s] defined", PROP_LOCAL_TEXT_EXTRACTION_DIR, REPOSITORY_HOME);
+ if (textExtractionDir == null) {
+ throw new IllegalStateException(String.format("Text extraction directory cannot be determined as neither " +
+ "directory path [%s] nor repository home [%s] defined", PROP_LOCAL_TEXT_EXTRACTION_DIR, REPOSITORY_HOME));
+ }
this.textExtractionDir = new File(textExtractionDir);
}
- private void registerExtractedTextProvider(PreExtractedTextProvider provider){
- if (extractedTextCache != null){
- if (provider != null){
+ private void registerExtractedTextProvider(PreExtractedTextProvider provider) {
+ if (extractedTextCache != null) {
+ if (provider != null) {
String usage = extractedTextCache.isAlwaysUsePreExtractedCache() ?
"always" : "only during reindexing phase";
LOG.info("Registering PreExtractedTextProvider {} with extracted text cache. " +
- "It would be used {}", provider, usage);
+ "It would be used {}", provider, usage);
} else {
LOG.info("Unregistering PreExtractedTextProvider with extracted text cache");
}
@@ -238,16 +241,35 @@ public class ElasticsearchIndexProviderS
}
}
- private ElasticsearchIndexCoordinateFactory getElasticsearchIndexCoordinateFactory(Map<String, ?> config) {
- ElasticsearchIndexCoordinateFactory esIndexCoordFactory;
- Map<String, String> esCfg = Maps.newHashMap();
- esCfg.put(ElasticsearchCoordinate.SCHEME_PROP,
- PropertiesUtil.toString(config.get(PROP_ELASTICSEARCH_SCHEME), PROP_ELASTICSEARCH_SCHEME_DEFAULT));
- esCfg.put(ElasticsearchCoordinate.HOST_PROP,
- PropertiesUtil.toString(config.get(PROP_ELASTICSEARCH_HOST), PROP_ELASTICSEARCH_HOST_DEFAULT));
- esCfg.put(ElasticsearchCoordinate.PORT_PROP, String.valueOf(
- PropertiesUtil.toInteger(config.get(PROP_ELASTICSEARCH_PORT), PROP_ELASTICSEARCH_PORT_DEFAULT)));
- esIndexCoordFactory = new DefaultElasticsearchIndexCoordinateFactory(connectionFactory, esCfg);
- return esIndexCoordFactory;
+ private ElasticsearchConnection getElasticsearchCoordinate(Map<String, ?> contextConfig) {
+ // system properties have priority
+ ElasticsearchConnection connection = build(System.getProperties().entrySet()
+ .stream()
+ .collect(Collectors.toMap(
+ e -> String.valueOf(e.getKey()),
+ e -> String.valueOf(e.getValue()))
+ )
+ );
+
+ if (connection == null) {
+ connection = build(contextConfig);
+ }
+
+ return connection != null ? connection : ElasticsearchConnection.defaultConnection.get();
+ }
+
+ private ElasticsearchConnection build(@NotNull Map<String, ?> config) {
+ ElasticsearchConnection coordinate = null;
+ Object p = config.get(PROP_ELASTICSEARCH_PORT);
+ if (p != null) {
+ try {
+ Integer port = Integer.parseInt(p.toString());
+ coordinate = new ElasticsearchConnection((String) config.get(PROP_ELASTICSEARCH_SCHEME),
+ (String) config.get(PROP_ELASTICSEARCH_HOST), port);
+ } catch (NumberFormatException nfe) {
+ LOG.warn("{} value ({}) cannot be parsed to a valid number", PROP_ELASTICSEARCH_PORT, p);
+ }
+ }
+ return coordinate;
}
}
Modified: jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocument.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocument.java?rev=1875701&r1=1875700&r2=1875701&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocument.java (original)
+++ jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchDocument.java Thu Mar 26 11:02:41 2020
@@ -16,7 +16,6 @@
*/
package org.apache.jackrabbit.oak.plugins.index.elasticsearch.index;
-import com.google.common.collect.Maps;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
import org.elasticsearch.common.Strings;
@@ -28,11 +27,11 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import static com.google.common.collect.Lists.newArrayList;
-
public class ElasticsearchDocument {
private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchDocument.class);
@@ -55,11 +54,11 @@ public class ElasticsearchDocument {
LOG.warn("Couldn't encode {} as ES id", path);
}
this.id = id;
- this.fulltext = newArrayList();
- this.suggest = newArrayList();
- this.notNullProps = newArrayList();
- this.nullProps = newArrayList();
- this.properties = Maps.newHashMap();
+ this.fulltext = new ArrayList<>();
+ this.suggest = new ArrayList<>();
+ this.notNullProps = new ArrayList<>();
+ this.nullProps = new ArrayList<>();
+ this.properties = new HashMap<>();
}
void addFulltext(String value) {
Modified: jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorContext.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorContext.java?rev=1875701&r1=1875700&r2=1875701&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorContext.java (original)
+++ jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorContext.java Thu Mar 26 11:02:41 2020
@@ -26,6 +26,8 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.jetbrains.annotations.Nullable;
+import java.io.IOException;
+
public class ElasticsearchIndexEditorContext extends FulltextIndexEditorContext<ElasticsearchDocument> {
ElasticsearchIndexEditorContext(NodeState root,
NodeBuilder definition, @Nullable IndexDefinition indexDefinition,
@@ -53,11 +55,15 @@ public class ElasticsearchIndexEditorCon
// Now, that index definition _might_ have been migrated by super call, it would be ok to
// get writer and provision index settings and mappings
- getWriter().setProvisioningRequired();
+ try {
+ getWriter().provisionIndex();
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to provision index", e);
+ }
}
@Override
public ElasticsearchIndexWriter getWriter() {
- return (ElasticsearchIndexWriter)super.getWriter();
+ return (ElasticsearchIndexWriter) super.getWriter();
}
}
\ No newline at end of file
Modified: jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorProvider.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorProvider.java?rev=1875701&r1=1875700&r2=1875701&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorProvider.java (original)
+++ jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexEditorProvider.java Thu Mar 26 11:02:41 2020
@@ -21,7 +21,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback;
import org.apache.jackrabbit.oak.plugins.index.IndexingContext;
-import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection;
import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDefinition;
import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
import org.apache.jackrabbit.oak.spi.commit.Editor;
@@ -30,17 +30,16 @@ import org.apache.jackrabbit.oak.spi.sta
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexConstants.TYPE_ELASTICSEARCH;
public class ElasticsearchIndexEditorProvider implements IndexEditorProvider {
- private final ElasticsearchIndexCoordinateFactory esIndexCoordFactory;
+ private final ElasticsearchConnection elasticsearchConnection;
private final ExtractedTextCache extractedTextCache;
- public ElasticsearchIndexEditorProvider(@NotNull ElasticsearchIndexCoordinateFactory esIndexCoordFactory,
+ public ElasticsearchIndexEditorProvider(@NotNull ElasticsearchConnection elasticsearchConnection,
ExtractedTextCache extractedTextCache) {
- this.esIndexCoordFactory = esIndexCoordFactory;
+ this.elasticsearchConnection = elasticsearchConnection;
this.extractedTextCache = extractedTextCache != null ? extractedTextCache : new ExtractedTextCache(0, 0);
}
@@ -49,15 +48,16 @@ public class ElasticsearchIndexEditorPro
@NotNull NodeBuilder definition, @NotNull NodeState root,
@NotNull IndexUpdateCallback callback) throws CommitFailedException {
if (TYPE_ELASTICSEARCH.equals(type)) {
- checkArgument(callback instanceof ContextAwareCallback, "callback instance not of type " +
- "ContextAwareCallback [%s]", callback);
- IndexingContext indexingContext = ((ContextAwareCallback)callback).getIndexingContext();
+ if (!(callback instanceof ContextAwareCallback)) {
+ throw new IllegalStateException("callback instance not of type ContextAwareCallback [" + callback + "]");
+ }
+ IndexingContext indexingContext = ((ContextAwareCallback) callback).getIndexingContext();
String indexPath = indexingContext.getIndexPath();
ElasticsearchIndexDefinition indexDefinition =
new ElasticsearchIndexDefinition(root, definition.getNodeState(), indexPath);
- ElasticsearchIndexWriterFactory writerFactory = new ElasticsearchIndexWriterFactory(esIndexCoordFactory);
+ ElasticsearchIndexWriterFactory writerFactory = new ElasticsearchIndexWriterFactory(elasticsearchConnection);
ElasticsearchIndexEditorContext context = new ElasticsearchIndexEditorContext(root,
definition, indexDefinition,
Modified: jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriter.java?rev=1875701&r1=1875700&r2=1875701&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriter.java (original)
+++ jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriter.java Thu Mar 26 11:02:41 2020
@@ -16,8 +16,8 @@
*/
package org.apache.jackrabbit.oak.plugins.index.elasticsearch.index;
-import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory;
-import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinate;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDescriptor;
import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
import org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexWriter;
@@ -26,7 +26,6 @@ import org.elasticsearch.action.delete.D
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.common.Strings;
@@ -49,119 +48,97 @@ import static org.elasticsearch.common.x
public class ElasticsearchIndexWriter implements FulltextIndexWriter<ElasticsearchDocument> {
private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchIndexWriter.class);
- private final ElasticsearchIndexCoordinate esIndexCoord;
- private final RestHighLevelClient client;
- private boolean shouldProvisionIndex;
+ private final ElasticsearchIndexDescriptor indexDescriptor;
private final boolean isAsync;
// TODO: use bulk API - https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-docs-bulk-processor.html
ElasticsearchIndexWriter(@NotNull IndexDefinition indexDefinition,
- ElasticsearchIndexCoordinateFactory esIndexCoordFactory) {
- esIndexCoord = esIndexCoordFactory.getElasticsearchIndexCoordinate(indexDefinition);
- client = esIndexCoord.getClient();
+ @NotNull ElasticsearchConnection elasticsearchConnection) {
+ indexDescriptor = new ElasticsearchIndexDescriptor(elasticsearchConnection, indexDefinition);
// TODO: ES indexing put another bit delay before docs appear in search.
// For test without "async" indexing, we can use following hack BUT those where we
// would setup async, we'd need to find another way.
isAsync = indexDefinition.getDefinitionNodeState().getProperty("async") != null;
-
- shouldProvisionIndex = false;
}
@Override
public void updateDocument(String path, ElasticsearchDocument doc) throws IOException {
- provisionIndex();
- IndexRequest request = new IndexRequest(esIndexCoord.getEsIndexName())
+ IndexRequest request = new IndexRequest(indexDescriptor.getIndexName())
.id(pathToId(path))
// immediate refresh would slow indexing response such that next
// search would see the effect of this indexed doc. Must only get
// enabled in tests (hopefully there are no non-async indexes in real life)
.setRefreshPolicy(isAsync ? NONE : IMMEDIATE)
.source(doc.build(), XContentType.JSON);
- IndexResponse response = client.index(request, RequestOptions.DEFAULT);
+ IndexResponse response = indexDescriptor.getClient().index(request, RequestOptions.DEFAULT);
LOG.trace("update {} - {}. Response: {}", path, doc, response);
}
@Override
public void deleteDocuments(String path) throws IOException {
- provisionIndex();
- DeleteRequest request = new DeleteRequest(esIndexCoord.getEsIndexName())
+ DeleteRequest request = new DeleteRequest(indexDescriptor.getIndexName())
.id(pathToId(path))
// immediate refresh would slow indexing response such that next
// search would see the effect of this indexed doc. Must only get
// enabled in tests (hopefully there are no non-async indexes in real life)
.setRefreshPolicy(isAsync ? NONE : IMMEDIATE);
- DeleteResponse response = client.delete(request, RequestOptions.DEFAULT);
+ DeleteResponse response = indexDescriptor.getClient().delete(request, RequestOptions.DEFAULT);
LOG.trace("delete {}. Response: {}", path, response);
}
@Override
public boolean close(long timestamp) throws IOException {
- provisionIndex();
// TODO : track index updates and return accordingly
// TODO : if/when we do async push, this is where to wait for those ops to complete
return false;
}
- /**
- * This method <b>won't</b> immediately provision index. But, provision would be done <b>before</b>
- * any updates are sent to the index
- */
- void setProvisioningRequired() {
- shouldProvisionIndex = true;
- }
-
- private void provisionIndex() throws IOException {
- if (!shouldProvisionIndex) {
- return;
- }
-
- try {
- CreateIndexRequest request = new CreateIndexRequest(esIndexCoord.getEsIndexName());
-
- // provision settings
- request.settings(Settings.builder()
- .put("analysis.analyzer.ancestor_analyzer.type", "custom")
- .put("analysis.analyzer.ancestor_analyzer.tokenizer", "path_hierarchy"));
-
- // provision mappings
- XContentBuilder mappingBuilder = XContentFactory.jsonBuilder();
- mappingBuilder.startObject();
+ // TODO: we need to check if the index already exists and in that case we have to figure out if there are
+ // "breaking changes" in the index definition
+ protected void provisionIndex() throws IOException {
+ CreateIndexRequest request = new CreateIndexRequest(indexDescriptor.getIndexName());
+
+ // provision settings
+ request.settings(Settings.builder()
+ .put("analysis.analyzer.ancestor_analyzer.type", "custom")
+ .put("analysis.analyzer.ancestor_analyzer.tokenizer", "path_hierarchy"));
+
+ // provision mappings
+ XContentBuilder mappingBuilder = XContentFactory.jsonBuilder();
+ mappingBuilder.startObject();
+ {
+ mappingBuilder.startObject("properties");
{
- mappingBuilder.startObject("properties");
- {
- mappingBuilder.startObject(FieldNames.ANCESTORS)
- .field("type", "text")
- .field("analyzer", "ancestor_analyzer")
- .field("search_analyzer", "keyword")
- .field("search_quote_analyzer", "keyword")
- .endObject();
- mappingBuilder.startObject(FieldNames.PATH_DEPTH)
- .field("type", "integer")
- .endObject();
- mappingBuilder.startObject(FieldNames.SUGGEST)
- .field("type", "completion")
- .endObject();
- mappingBuilder.startObject(FieldNames.NOT_NULL_PROPS)
- .field("type", "keyword")
- .endObject();
- mappingBuilder.startObject(FieldNames.NULL_PROPS)
- .field("type", "keyword")
- .endObject();
- }
- mappingBuilder.endObject();
+ mappingBuilder.startObject(FieldNames.ANCESTORS)
+ .field("type", "text")
+ .field("analyzer", "ancestor_analyzer")
+ .field("search_analyzer", "keyword")
+ .field("search_quote_analyzer", "keyword")
+ .endObject();
+ mappingBuilder.startObject(FieldNames.PATH_DEPTH)
+ .field("type", "integer")
+ .endObject();
+ mappingBuilder.startObject(FieldNames.SUGGEST)
+ .field("type", "completion")
+ .endObject();
+ mappingBuilder.startObject(FieldNames.NOT_NULL_PROPS)
+ .field("type", "keyword")
+ .endObject();
+ mappingBuilder.startObject(FieldNames.NULL_PROPS)
+ .field("type", "keyword")
+ .endObject();
}
mappingBuilder.endObject();
- request.mapping(mappingBuilder);
+ }
+ mappingBuilder.endObject();
+ request.mapping(mappingBuilder);
- String requestMsg = Strings.toString(request.toXContent(jsonBuilder(), EMPTY_PARAMS));
- CreateIndexResponse response = client.indices().create(request, RequestOptions.DEFAULT);
+ String requestMsg = Strings.toString(request.toXContent(jsonBuilder(), EMPTY_PARAMS));
+ CreateIndexResponse response = indexDescriptor.getClient().indices().create(request, RequestOptions.DEFAULT);
- LOG.info("Updated settings {}. Response acknowledged: {}", requestMsg, response.isAcknowledged());
- } finally {
- shouldProvisionIndex = false;
- }
+ LOG.info("Updated settings {}. Response acknowledged: {}", requestMsg, response.isAcknowledged());
}
}
Modified: jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriterFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriterFactory.java?rev=1875701&r1=1875700&r2=1875701&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriterFactory.java (original)
+++ jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/index/ElasticsearchIndexWriterFactory.java Thu Mar 26 11:02:41 2020
@@ -16,21 +16,21 @@
*/
package org.apache.jackrabbit.oak.plugins.index.elasticsearch.index;
-import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
import org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexWriterFactory;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.jetbrains.annotations.NotNull;
public class ElasticsearchIndexWriterFactory implements FulltextIndexWriterFactory<ElasticsearchDocument> {
- private final ElasticsearchIndexCoordinateFactory esIndexCoordFactory;
+ private final ElasticsearchConnection elasticsearchConnection;
- ElasticsearchIndexWriterFactory(@NotNull ElasticsearchIndexCoordinateFactory esIndexCoordFactory) {
- this.esIndexCoordFactory = esIndexCoordFactory;
+ ElasticsearchIndexWriterFactory(@NotNull ElasticsearchConnection elasticsearchConnection) {
+ this.elasticsearchConnection = elasticsearchConnection;
}
@Override
public ElasticsearchIndexWriter newInstance(IndexDefinition definition, NodeBuilder definitionBuilder, boolean reindex) {
- return new ElasticsearchIndexWriter(definition, esIndexCoordFactory);
+ return new ElasticsearchIndexWriter(definition, elasticsearchConnection);
}
}
Modified: jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java?rev=1875701&r1=1875700&r2=1875701&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java (original)
+++ jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndex.java Thu Mar 26 11:02:41 2020
@@ -16,12 +16,12 @@
*/
package org.apache.jackrabbit.oak.plugins.index.elasticsearch.query;
-import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection;
import org.apache.jackrabbit.oak.plugins.index.search.IndexNode;
import org.apache.jackrabbit.oak.plugins.index.search.SizeEstimator;
-import org.apache.jackrabbit.oak.plugins.index.search.util.LMSEstimator;
import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndex;
import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndexPlanner;
+import org.apache.jackrabbit.oak.plugins.index.search.util.LMSEstimator;
import org.apache.jackrabbit.oak.spi.query.Cursor;
import org.apache.jackrabbit.oak.spi.query.Filter;
import org.apache.jackrabbit.oak.spi.query.QueryLimits;
@@ -45,11 +45,11 @@ public class ElasticsearchIndex extends
// higher than some threshold below which the query should rather be answered by something else if possible
private static final double MIN_COST = 100.1;
- private final ElasticsearchIndexCoordinateFactory esIndexCoordFactory;
+ private final ElasticsearchConnection elasticsearchConnection;
private final NodeState root;
- ElasticsearchIndex(@NotNull ElasticsearchIndexCoordinateFactory esIndexCoordFactory, @NotNull NodeState root) {
- this.esIndexCoordFactory = esIndexCoordFactory;
+ ElasticsearchIndex(@NotNull ElasticsearchConnection elasticsearchConnection, @NotNull NodeState root) {
+ this.elasticsearchConnection = elasticsearchConnection;
this.root = root;
}
@@ -85,9 +85,7 @@ public class ElasticsearchIndex extends
@Override
protected IndexNode acquireIndexNode(String indexPath) {
- ElasticsearchIndexNode elasticsearchIndexNode = ElasticsearchIndexNode.fromIndexPath(root, indexPath);
- elasticsearchIndexNode.setFactory(esIndexCoordFactory);
- return elasticsearchIndexNode;
+ return new ElasticsearchIndexNode(root, indexPath, elasticsearchConnection);
}
@Override
@@ -104,7 +102,7 @@ public class ElasticsearchIndex extends
final FulltextIndexPlanner.PlanResult pr = getPlanResult(plan);
QueryLimits settings = filter.getQueryLimits();
- Iterator<FulltextResultRow> itr = new ElasticsearchResultRowIterator(esIndexCoordFactory, filter, pr, plan,
+ Iterator<FulltextResultRow> itr = new ElasticsearchResultRowIterator(filter, pr, plan,
acquireIndexNode(plan), FulltextIndex::shouldInclude, getEstimator(plan.getPlanName()));
SizeEstimator sizeEstimator = getSizeEstimator(plan);
Modified: jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexNode.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexNode.java?rev=1875701&r1=1875700&r2=1875701&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexNode.java (original)
+++ jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexNode.java Thu Mar 26 11:02:41 2020
@@ -16,8 +16,9 @@
*/
package org.apache.jackrabbit.oak.plugins.index.elasticsearch.query;
-import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection;
import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDescriptor;
import org.apache.jackrabbit.oak.plugins.index.search.IndexNode;
import org.apache.jackrabbit.oak.plugins.index.search.IndexStatistics;
import org.apache.jackrabbit.oak.spi.state.NodeState;
@@ -28,16 +29,13 @@ import org.jetbrains.annotations.Nullabl
public class ElasticsearchIndexNode implements IndexNode {
private final ElasticsearchIndexDefinition indexDefinition;
- private ElasticsearchIndexCoordinateFactory factory;
+ private final ElasticsearchIndexDescriptor indexDescriptor;
- static ElasticsearchIndexNode fromIndexPath(@NotNull NodeState root, @NotNull String indexPath) {
- NodeState indexNS = NodeStateUtils.getNode(root, indexPath);
- ElasticsearchIndexDefinition indexDefinition = new ElasticsearchIndexDefinition(root, indexNS, indexPath);
- return new ElasticsearchIndexNode(indexDefinition);
- }
-
- private ElasticsearchIndexNode(ElasticsearchIndexDefinition indexDefinition) {
- this.indexDefinition = indexDefinition;
+ protected ElasticsearchIndexNode(@NotNull NodeState root, @NotNull String indexPath,
+ @NotNull ElasticsearchConnection elasticsearchConnection) {
+ final NodeState indexNS = NodeStateUtils.getNode(root, indexPath);
+ this.indexDefinition = new ElasticsearchIndexDefinition(root, indexNS, indexPath);
+ this.indexDescriptor = new ElasticsearchIndexDescriptor(elasticsearchConnection, indexDefinition);
}
@Override
@@ -50,6 +48,10 @@ public class ElasticsearchIndexNode impl
return indexDefinition;
}
+ public ElasticsearchIndexDescriptor getIndexDescriptor() {
+ return indexDescriptor;
+ }
+
@Override
public int getIndexNodeId() {
// TODO: does it matter that we simply return 0 as there's no observation based _refresh_ going on here
@@ -59,10 +61,6 @@ public class ElasticsearchIndexNode impl
@Override
public @Nullable IndexStatistics getIndexStatistics() {
- return new ElasticsearchIndexStatistics(factory.getElasticsearchIndexCoordinate(indexDefinition));
- }
-
- public void setFactory(ElasticsearchIndexCoordinateFactory factory) {
- this.factory = factory;
+ return new ElasticsearchIndexStatistics(indexDescriptor);
}
}
Modified: jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexProvider.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexProvider.java?rev=1875701&r1=1875700&r2=1875701&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexProvider.java (original)
+++ jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexProvider.java Thu Mar 26 11:02:41 2020
@@ -16,24 +16,24 @@
*/
package org.apache.jackrabbit.oak.plugins.index.elasticsearch.query;
-import com.google.common.collect.ImmutableList;
-import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchConnection;
import org.apache.jackrabbit.oak.spi.query.QueryIndex;
import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.jetbrains.annotations.NotNull;
+import java.util.Collections;
import java.util.List;
public class ElasticsearchIndexProvider implements QueryIndexProvider {
- private final ElasticsearchIndexCoordinateFactory esIndexCoordFactory;
+ private final ElasticsearchConnection elasticsearchConnection;
- public ElasticsearchIndexProvider(@NotNull ElasticsearchIndexCoordinateFactory esIndexCoordFactory) {
- this.esIndexCoordFactory = esIndexCoordFactory;
+ public ElasticsearchIndexProvider(ElasticsearchConnection elasticsearchConnection) {
+ this.elasticsearchConnection = elasticsearchConnection;
}
@Override
public @NotNull List<? extends QueryIndex> getQueryIndexes(NodeState nodeState) {
- return ImmutableList.of(new ElasticsearchIndex(esIndexCoordFactory, nodeState));
+ return Collections.singletonList(new ElasticsearchIndex(elasticsearchConnection, nodeState));
}
}
Modified: jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexStatistics.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexStatistics.java?rev=1875701&r1=1875700&r2=1875701&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexStatistics.java (original)
+++ jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchIndexStatistics.java Thu Mar 26 11:02:41 2020
@@ -16,31 +16,28 @@
*/
package org.apache.jackrabbit.oak.plugins.index.elasticsearch.query;
-import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinate;
+import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDescriptor;
import org.apache.jackrabbit.oak.plugins.index.search.IndexStatistics;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
import java.io.IOException;
public class ElasticsearchIndexStatistics implements IndexStatistics {
- private final ElasticsearchIndexCoordinate elasticsearchIndexCoordinate;
+ private final ElasticsearchIndexDescriptor elasticsearchIndexDescriptor;
- ElasticsearchIndexStatistics(ElasticsearchIndexCoordinate elasticsearchIndexCoordinate) {
- this.elasticsearchIndexCoordinate = elasticsearchIndexCoordinate;
+ ElasticsearchIndexStatistics(ElasticsearchIndexDescriptor elasticsearchIndexDescriptor) {
+ this.elasticsearchIndexDescriptor = elasticsearchIndexDescriptor;
}
@Override
public int numDocs() {
CountRequest countRequest = new CountRequest();
- SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
- searchSourceBuilder.query(QueryBuilders.matchAllQuery());
- countRequest.source(searchSourceBuilder);
+ countRequest.query(QueryBuilders.matchAllQuery());
try {
- CountResponse count = elasticsearchIndexCoordinate.getClient().count(countRequest, RequestOptions.DEFAULT);
+ CountResponse count = elasticsearchIndexDescriptor.getClient().count(countRequest, RequestOptions.DEFAULT);
return (int) count.getCount();
} catch (IOException e) {
// ignore failure
@@ -51,11 +48,9 @@ public class ElasticsearchIndexStatistic
@Override
public int getDocCountFor(String key) {
CountRequest countRequest = new CountRequest();
- SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
- searchSourceBuilder.query(QueryBuilders.existsQuery(key));
- countRequest.source(searchSourceBuilder);
+ countRequest.query(QueryBuilders.existsQuery(key));
try {
- CountResponse count = elasticsearchIndexCoordinate.getClient().count(countRequest, RequestOptions.DEFAULT);
+ CountResponse count = elasticsearchIndexDescriptor.getClient().count(countRequest, RequestOptions.DEFAULT);
return (int) count.getCount();
} catch (IOException e) {
// ignore failure
Modified: jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java?rev=1875701&r1=1875700&r2=1875701&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java (original)
+++ jackrabbit/oak/trunk/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elasticsearch/query/ElasticsearchResultRowIterator.java Thu Mar 26 11:02:41 2020
@@ -16,24 +16,25 @@
*/
package org.apache.jackrabbit.oak.plugins.index.elasticsearch.query;
-import com.google.common.collect.AbstractIterator;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Queues;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.commons.PerfLogger;
-import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexCoordinateFactory;
import org.apache.jackrabbit.oak.plugins.index.elasticsearch.ElasticsearchIndexDefinition;
import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition;
-import org.apache.jackrabbit.oak.plugins.index.search.util.LMSEstimator;
import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndex;
import org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndexPlanner.PlanResult;
+import org.apache.jackrabbit.oak.plugins.index.search.util.LMSEstimator;
import org.apache.jackrabbit.oak.spi.query.Filter;
import org.apache.jackrabbit.oak.spi.query.QueryConstants;
import org.apache.jackrabbit.oak.spi.query.QueryIndex.IndexPlan;
-import org.apache.jackrabbit.oak.spi.query.fulltext.*;
+import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextAnd;
+import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextContains;
+import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextExpression;
+import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextOr;
+import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextTerm;
+import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextVisitor;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
@@ -48,25 +49,41 @@ import javax.jcr.PropertyType;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
+import java.util.Iterator;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiPredicate;
+import java.util.stream.StreamSupport;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkState;
import static org.apache.jackrabbit.JcrConstants.JCR_MIXINTYPES;
import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE;
import static org.apache.jackrabbit.oak.api.Type.STRING;
import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot;
import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath;
-import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.*;
+import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newAncestorQuery;
+import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newDepthQuery;
+import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newMixinTypeQuery;
+import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newNodeTypeQuery;
+import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newNotNullPropQuery;
+import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newNullPropQuery;
+import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newPathQuery;
+import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newPrefixPathQuery;
+import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newPrefixQuery;
+import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newPropertyRestrictionQuery;
+import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newWildcardPathQuery;
+import static org.apache.jackrabbit.oak.plugins.index.elasticsearch.util.TermQueryBuilderFactory.newWildcardQuery;
import static org.apache.jackrabbit.oak.plugins.index.search.spi.query.FulltextIndex.isNodePath;
import static org.apache.jackrabbit.oak.spi.query.QueryConstants.JCR_PATH;
import static org.apache.jackrabbit.util.ISO8601.parse;
-import static org.elasticsearch.index.query.QueryBuilders.*;
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-public class ElasticsearchResultRowIterator extends AbstractIterator<FulltextIndex.FulltextResultRow> {
+public class ElasticsearchResultRowIterator implements Iterator<FulltextIndex.FulltextResultRow> {
private static final Logger LOG = LoggerFactory
.getLogger(ElasticsearchResultRowIterator.class);
private static final PerfLogger PERF_LOGGER =
@@ -83,15 +100,13 @@ public class ElasticsearchResultRowItera
private static final int ELASTICSEARCH_QUERY_MAX_BATCH_SIZE = 10000;
-
- private final Deque<FulltextIndex.FulltextResultRow> queue = Queues.newArrayDeque();
+ private final Deque<FulltextIndex.FulltextResultRow> queue = new ArrayDeque<>();
// TODO : find if ES can return dup docs - if so how to avoid
// private final Set<String> seenPaths = Sets.newHashSet();
private SearchHit lastDoc;
private int nextBatchSize = ELASTICSEARCH_QUERY_BATCH_SIZE;
private boolean noDocs = false;
- private final ElasticsearchIndexCoordinateFactory esIndexCoordFactory;
private final Filter filter;
private final PlanResult pr;
private final IndexPlan plan;
@@ -99,14 +114,12 @@ public class ElasticsearchResultRowItera
private final RowInclusionPredicate rowInclusionPredicate;
private final LMSEstimator estimator;
- ElasticsearchResultRowIterator(@NotNull ElasticsearchIndexCoordinateFactory esIndexCoordFactory,
- @NotNull Filter filter,
+ ElasticsearchResultRowIterator(@NotNull Filter filter,
@NotNull PlanResult pr,
@NotNull IndexPlan plan,
ElasticsearchIndexNode indexNode,
RowInclusionPredicate rowInclusionPredicate,
LMSEstimator estimator) {
- this.esIndexCoordFactory = esIndexCoordFactory;
this.filter = filter;
this.pr = pr;
this.plan = plan;
@@ -116,16 +129,18 @@ public class ElasticsearchResultRowItera
}
@Override
- protected FulltextIndex.FulltextResultRow computeNext() {
- if (!queue.isEmpty() || loadDocs()) {
- return queue.remove();
- } else {
- return endOfData();
- }
+ public boolean hasNext() {
+ return !queue.isEmpty() || loadDocs();
+ }
+
+ @Override
+ public FulltextIndex.FulltextResultRow next() {
+ return queue.remove();
}
/**
* Loads the lucene documents in batches
+ *
* @return true if any document is loaded
*/
private boolean loadDocs() {
@@ -134,9 +149,11 @@ public class ElasticsearchResultRowItera
return false;
}
- SearchHit lastDocToRecord = null;
+ if (indexNode == null) {
+ throw new IllegalStateException("indexNode cannot be null");
+ }
- checkState(indexNode != null);
+ SearchHit lastDocToRecord = null;
try {
ElasticsearchSearcher searcher = getCurrentSearcher(indexNode);
QueryBuilder query = getESRequest(plan, pr);
@@ -202,7 +219,7 @@ public class ElasticsearchResultRowItera
}
private ElasticsearchSearcher getCurrentSearcher(ElasticsearchIndexNode indexNode) {
- return new ElasticsearchSearcher(esIndexCoordFactory, indexNode);
+ return new ElasticsearchSearcher(indexNode);
}
private FulltextIndex.FulltextResultRow convertToRow(SearchHit hit) throws IOException {
@@ -240,7 +257,7 @@ public class ElasticsearchResultRowItera
/**
* Get the Elasticsearch query for the given filter.
*
- * @param plan index plan containing filter details
+ * @param plan index plan containing filter details
* @param planResult
* @return the Lucene query
*/
@@ -329,7 +346,7 @@ public class ElasticsearchResultRowItera
if (x instanceof BoolQueryBuilder) {
BoolQueryBuilder bq = (BoolQueryBuilder) x;
if (bq.mustNot().size() == 1
- // no other clauses
+ // no other clauses
&& bq.should().isEmpty() && bq.must().isEmpty() && bq.filter().isEmpty()) {
hasMustNot = true;
q.mustNot(bq.mustNot().get(0));
@@ -352,9 +369,6 @@ public class ElasticsearchResultRowItera
private boolean visitTerm(String propertyName, String text, String boost, boolean not) {
String p = getLuceneFieldName(propertyName, pr);
QueryBuilder q = tokenToQuery(text, p, pr);
- if (q == null) {
- return false;
- }
if (boost != null) {
q.boost(Float.parseFloat(boost));
}
@@ -430,7 +444,6 @@ public class ElasticsearchResultRowItera
*/
@NotNull
private static QueryBuilder performAdditionalWraps(@NotNull List<QueryBuilder> qs) {
- checkNotNull(qs);
if (qs.size() == 1) {
// we don't need to worry about all-negatives in a bool query as
// BoolQueryBuilder.adjustPureNegative is on by default anyway
@@ -455,13 +468,11 @@ public class ElasticsearchResultRowItera
/**
* unwraps any NOT clauses from the provided boolean query into another boolean query.
*
- * @param input the query to be analysed for the existence of NOT clauses. Cannot be null.
+ * @param input the query to be analysed for the existence of NOT clauses. Cannot be null.
* @param output the query where the unwrapped NOTs will be saved into. Cannot be null.
* @return true if there where at least one unwrapped NOT. false otherwise.
*/
private static boolean unwrapMustNot(@NotNull BoolQueryBuilder input, @NotNull BoolQueryBuilder output) {
- checkNotNull(input);
- checkNotNull(output);
boolean unwrapped = false;
for (QueryBuilder mustNot : input.mustNot()) {
output.mustNot(mustNot);
@@ -480,6 +491,9 @@ public class ElasticsearchResultRowItera
private static void addNonFullTextConstraints(List<QueryBuilder> qs,
IndexPlan plan, PlanResult planResult) {
+ final BiPredicate<Iterable<String>, String> any = (iterable, value) ->
+ StreamSupport.stream(iterable.spliterator(), false).anyMatch(value::equals);
+
Filter filter = plan.getFilter();
IndexDefinition defn = planResult.indexDefinition;
if (!filter.matchesAllTypes()) {
@@ -509,7 +523,7 @@ public class ElasticsearchResultRowItera
// deduced
if (planResult.isPathTransformed()) {
String parentPathSegment = planResult.getParentPathSegment();
- if ( ! Iterables.any(PathUtils.elements(parentPathSegment), "*"::equals)) {
+ if (!any.test(PathUtils.elements(parentPathSegment), "*")) {
qs.add(newPathQuery(path + parentPathSegment));
}
} else {
@@ -527,7 +541,7 @@ public class ElasticsearchResultRowItera
// deduced
if (planResult.isPathTransformed()) {
String parentPathSegment = planResult.getParentPathSegment();
- if ( ! Iterables.any(PathUtils.elements(parentPathSegment), "*"::equals)) {
+ if (!any.test(PathUtils.elements(parentPathSegment), "*")) {
qs.add(newPathQuery(getParentPath(path) + parentPathSegment));
}
} else {
@@ -657,7 +671,7 @@ public class ElasticsearchResultRowItera
@Nullable
private static QueryBuilder createQuery(String propertyName, Filter.PropertyRestriction pr,
- PropertyDefinition defn) {
+ PropertyDefinition defn) {
int propType = FulltextIndex.determinePropertyType(defn, pr);
if (pr.isNullRestriction()) {