You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2018/09/25 12:24:18 UTC
svn commit: r1841926 [2/14] - in /jackrabbit/oak/trunk:
oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/
oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/util/
oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/scal...
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java?rev=1841926&r1=1841925&r2=1841926&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java Tue Sep 25 12:24:15 2018
@@ -35,28 +35,31 @@ import com.google.common.collect.Sets;
import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.api.PropertyValue;
import org.apache.jackrabbit.oak.api.Result.SizePrecision;
-import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule;
+import org.apache.jackrabbit.oak.plugins.index.Cursors;
+import org.apache.jackrabbit.oak.plugins.index.Cursors.PathCursor;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.MoreLikeThisHelper;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.PathStoredFieldVisitor;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.SpellcheckHelper;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.SuggestHelper;
+import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.IndexingRule;
+import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.SizeEstimator;
import org.apache.jackrabbit.oak.plugins.memory.PropertyValues;
-import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextAnd;
-import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextContains;
-import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextExpression;
-import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextOr;
-import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextTerm;
-import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextVisitor;
import org.apache.jackrabbit.oak.spi.query.Cursor;
-import org.apache.jackrabbit.oak.plugins.index.Cursors;
-import org.apache.jackrabbit.oak.plugins.index.Cursors.PathCursor;
import org.apache.jackrabbit.oak.spi.query.Filter;
import org.apache.jackrabbit.oak.spi.query.Filter.PropertyRestriction;
import org.apache.jackrabbit.oak.spi.query.IndexRow;
import org.apache.jackrabbit.oak.spi.query.QueryConstants;
-import org.apache.jackrabbit.oak.spi.query.QueryIndex;
import org.apache.jackrabbit.oak.spi.query.QueryIndex.AdvanceFulltextQueryIndex;
import org.apache.jackrabbit.oak.spi.query.QueryLimits;
+import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextAnd;
+import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextContains;
+import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextExpression;
+import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextOr;
+import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextTerm;
+import org.apache.jackrabbit.oak.spi.query.fulltext.FullTextVisitor;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
@@ -69,7 +72,6 @@ import org.apache.lucene.index.MultiFiel
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
@@ -99,15 +101,23 @@ import static com.google.common.base.Pre
import static org.apache.jackrabbit.JcrConstants.JCR_MIXINTYPES;
import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE;
import static org.apache.jackrabbit.oak.api.Type.STRING;
-import static org.apache.jackrabbit.oak.commons.PathUtils.*;
+import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot;
+import static org.apache.jackrabbit.oak.commons.PathUtils.getAncestorPath;
+import static org.apache.jackrabbit.oak.commons.PathUtils.getDepth;
+import static org.apache.jackrabbit.oak.commons.PathUtils.getName;
+import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.VERSION;
import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newFulltextTerm;
import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newPathTerm;
-import static org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper.skipTokenization;
+import static org.apache.jackrabbit.oak.plugins.index.search.util.IndexHelper.skipTokenization;
import static org.apache.jackrabbit.oak.spi.query.QueryConstants.JCR_PATH;
-import static org.apache.lucene.search.BooleanClause.Occur.*;
+import static org.apache.lucene.search.BooleanClause.Occur.MUST;
+import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT;
+import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
/**
+ * Used to query old (compatVersion 1) Lucene indexes.
+ *
* Provides a QueryIndex that does lookups against a Lucene-based index
*
* <p>
@@ -129,6 +139,8 @@ import static org.apache.lucene.search.B
* </ul>
* <p>
* <pre>{@code
+ * <pre>{@code
+ * {
* {
* NodeBuilder index = root.child("oak:index");
* index.child("lucene")
@@ -195,7 +207,7 @@ public class LuceneIndex implements Adva
return Collections.emptyList();
}
- String indexPath = new LuceneIndexLookup(rootState).getOldFullTextIndexPath(filter, tracker);
+ String indexPath = LuceneIndexLookupUtil.getOldFullTextIndexPath(rootState, filter, tracker);
if (indexPath == null) { // unusable index
return Collections.emptyList();
}
@@ -206,7 +218,7 @@ public class LuceneIndex implements Adva
// "contains(a/x, 'hello') and contains(b/x, 'world')"
return Collections.emptyList();
}
- IndexNode node = tracker.acquireIndexNode(indexPath);
+ LuceneIndexNode node = tracker.acquireIndexNode(indexPath);
try{
if (node != null){
IndexDefinition defn = node.getDefinition();
@@ -241,7 +253,7 @@ public class LuceneIndex implements Adva
@Override
public String getPlanDescription(IndexPlan plan, NodeState root) {
Filter filter = plan.getFilter();
- IndexNode index = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH));
+ LuceneIndexNode index = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH));
checkState(index != null, "The Lucene index is not available");
try {
FullTextExpression ft = filter.getFullTextConstraint();
@@ -346,7 +358,7 @@ public class LuceneIndex implements Adva
ScoreDoc lastDocToRecord = null;
- IndexNode indexNode = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH));
+ LuceneIndexNode indexNode = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH));
checkState(indexNode != null);
try {
IndexSearcher searcher = indexNode.getSearcher();
@@ -480,7 +492,7 @@ public class LuceneIndex implements Adva
SizeEstimator sizeEstimator = new SizeEstimator() {
@Override
public long getSize() {
- IndexNode indexNode = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH));
+ LuceneIndexNode indexNode = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH));
checkState(indexNode != null);
try {
IndexSearcher searcher = indexNode.getSearcher();
@@ -602,8 +614,8 @@ public class LuceneIndex implements Adva
* @param indexDefinition nodestate that contains the index definition
* @return the Lucene query
*/
- private static LuceneRequestFacade getLuceneRequest(Filter filter, IndexReader reader,
- boolean nonFullTextConstraints, IndexDefinition indexDefinition) {
+ private static LuceneRequestFacade getLuceneRequest(Filter filter, IndexReader reader, boolean nonFullTextConstraints,
+ LuceneIndexDefinition indexDefinition) {
List<Query> qs = new ArrayList<Query>();
Analyzer analyzer = indexDefinition.getAnalyzer();
FullTextExpression ft = filter.getFullTextConstraint();
@@ -656,7 +668,7 @@ public class LuceneIndex implements Adva
}
private static void addNonFullTextConstraints(List<Query> qs,
- Filter filter, IndexReader reader, Analyzer analyzer, IndexDefinition indexDefinition) {
+ Filter filter, IndexReader reader, Analyzer analyzer, IndexDefinition indexDefinition) {
if (!filter.matchesAllTypes()) {
addNodeTypeConstraints(qs, filter);
}
@@ -807,7 +819,7 @@ public class LuceneIndex implements Adva
}
private static boolean isExcludedProperty(PropertyRestriction pr,
- IndexingRule rule) {
+ IndexingRule rule) {
String name = pr.propertyName;
if (name.contains("/")) {
// lucene cannot handle child-level property restrictions
@@ -904,7 +916,7 @@ public class LuceneIndex implements Adva
if (x instanceof BooleanQuery) {
BooleanQuery bq = (BooleanQuery) x;
if ((bq.getClauses().length == 1) &&
- (bq.getClauses()[0].getOccur() == BooleanClause.Occur.MUST_NOT)) {
+ (bq.getClauses()[0].getOccur() == Occur.MUST_NOT)) {
hasMustNot = true;
q.add(bq.getClauses()[0]);
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1841926&r1=1841925&r2=1841926&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java Tue Sep 25 12:24:15 2018
@@ -16,30 +16,18 @@
*/
package org.apache.jackrabbit.oak.plugins.index.lucene;
+import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
import org.apache.lucene.util.Version;
+/**
+ * Constants used internally in Lucene indexes.
+ */
public interface LuceneIndexConstants {
- enum IndexingMode {
- SYNC,
- NRT,
- ASYNC;
-
- public String asyncValueName(){
- return name().toLowerCase();
- }
-
- public static IndexingMode from(String indexingMode){
- return valueOf(indexingMode.toUpperCase());
- }
- }
-
String TYPE_LUCENE = "lucene";
- String INDEX_DATA_CHILD_NAME = ":data";
-
String SUGGEST_DATA_CHILD_NAME = ":suggest-data";
String TRASH_CHILD_NAME = ":trash";
@@ -49,66 +37,6 @@ public interface LuceneIndexConstants {
Analyzer ANALYZER = new OakAnalyzer(VERSION);
/**
- * include only certain property types in the index
- */
- String INCLUDE_PROPERTY_TYPES = "includePropertyTypes";
-
- /**
- * exclude certain properties by name
- */
- String EXCLUDE_PROPERTY_NAMES = "excludePropertyNames";
-
- String PERSISTENCE_NAME = "persistence";
-
- String PERSISTENCE_OAK = "repository";
-
- String PERSISTENCE_FILE = "file";
-
- String PERSISTENCE_PATH = "path";
-
- /**
- * Experimental flag to control storage behavior: 'null' or 'true' means the content is stored
- */
- String EXPERIMENTAL_STORAGE = "oak.experimental.storage";
-
- /**
- * Determines if full text indexing is enabled for this index definition.
- * Default is true
- */
- String FULL_TEXT_ENABLED = "fulltextEnabled";
-
- /**
- * Only include properties with name in this set. If this property is defined
- * then {@code excludePropertyNames} would be ignored
- */
- String INCLUDE_PROPERTY_NAMES = "includePropertyNames";
-
- /**
- * Type of the property being indexed defined as part of property definition
- * under the given index definition. Refer to {@link javax.jcr.PropertyType}
- * contants for the possible values
- */
- String PROP_TYPE = "type";
-
- /**
- * Defines properties which would be used for ordering. If range queries are to
- * be performed with same property then it must be part of include list also
- */
- String ORDERED_PROP_NAMES = "orderedProps";
-
- /**
- * Size in bytes used for splitting the index files when storing them in NodeStore
- */
- String BLOB_SIZE = "blobSize";
-
- /**
- * Native function name associated with this index definition. Any query can
- * use this as the function name to ensure that this index gets used for invoking
- * the index
- */
- String FUNC_NAME = "functionName";
-
- /**
* Name of the codec to be used for indexing
*/
String CODEC_NAME = "codec";
@@ -119,108 +47,11 @@ public interface LuceneIndexConstants {
String MERGE_POLICY_NAME = "mergePolicy";
/**
- * Child node name under which property details are provided
- */
- String PROP_NODE = "properties";
-
- String INDEX_RULES = "indexRules";
-
- /**
- * Field boost factor
- */
- String FIELD_BOOST = "boost";
-
- /**
- * Property name defined explicitly. Mostly used in case of relative property names
- */
- String PROP_NAME = "name";
-
- String PROP_IS_REGEX = "isRegexp";
-
- String PROP_INDEX = "index";
-
- String PROP_USE_IN_EXCERPT = "useInExcerpt";
-
- String EXCERPT_NODE_FIELD_NAME = ".";
-
- String PROP_NODE_SCOPE_INDEX = "nodeScopeIndex";
-
- String PROP_PROPERTY_INDEX = "propertyIndex";
-
- String PROP_ANALYZED = "analyzed";
-
- String RULE_INHERITED = "inherited";
-
- String PROP_ORDERED = "ordered";
-
- String PROP_SCORER_PROVIDER = "scorerProviderName";
-
- String PROP_WEIGHT = "weight";
-
- /**
- * Boolean property in property definition to mark sync properties
- */
- String PROP_SYNC = "sync";
-
- /**
- * Boolean property in property definition to mark unique properties
- */
- String PROP_UNIQUE = "unique";
-
- /**
- * Integer property indicating that LuceneIndex should be
- * used in compat mode to specific version
- */
- String COMPAT_MODE = "compatVersion";
-
- /**
* Boolean property to indicate that LuceneIndex is being used in testMode
* and it should participate in every test
*/
String TEST_MODE = "testMode";
- String EVALUATE_PATH_RESTRICTION = "evaluatePathRestrictions";
-
- /**
- * Experimental config to restrict which property type gets indexed at
- * property definition level. Mostly index rule level #INCLUDE_PROPERTY_TYPES
- * should be sufficient
- */
- String PROP_INCLUDED_TYPE = "oak.experimental.includePropertyTypes";
-
- /**
- * Regex to allow inclusion of all immediate properties of the node
- */
- String REGEX_ALL_PROPS = "^[^\\/]*$";
-
- /**
- * Node name storing the aggregate rules
- */
- String AGGREGATES = "aggregates";
-
- String AGG_PRIMARY_TYPE = "primaryType";
-
- /**
- * Name of property which stores the aggregate include pattern like <code>jcr:content/metadata</code>
- */
- String AGG_PATH = "path";
-
- /**
- * Limit for maximum number of reaggregates allowed. For example if there is an aggregate of nt:folder
- * and it also includes nt:folder then aggregation would traverse down untill this limit is hit
- */
- String AGG_RECURSIVE_LIMIT = "reaggregateLimit";
-
- /**
- * Boolean property indicating that separate fulltext field should be created for
- * node represented by this pattern
- */
- String AGG_RELATIVE_NODE = "relativeNode";
-
- String COST_PER_ENTRY = "costPerEntry";
-
- String COST_PER_EXECUTION = "costPerExecution";
-
/**
* Boolean property indicating if in-built analyzer should preserve original term
* (i.e. use
@@ -351,54 +182,4 @@ public interface LuceneIndexConstants {
@Deprecated
String INDEX_PATH = "indexPath";
- /**
- * Optional subnode holding configuration for facets.
- */
- String FACETS = "facets";
-
- /**
- * Optional property to set the suggest field to be analyzed and therefore allow more fine
- * grained and flexible suggestions.
- */
- String SUGGEST_ANALYZED = "suggestAnalyzed";
-
- /**
- * Optional (index definition) property indicating whether facets should be ACL checked.
- * Default is true
- */
- String PROP_SECURE_FACETS = "secure";
-
- /**
- * Optional (index definition) property indicating max number of facets that will be retrieved
- * in query
- * Default is {@link IndexDefinition#DEFAULT_FACET_COUNT}
- */
- String PROP_FACETS_TOP_CHILDREN = "topChildren";
-
- /**
- * Optional (property definition) property indicating whether facets should be created
- * for this property
- */
- String PROP_FACETS = "facets";
-
- /**
- * Boolean property indicate that property should not be included in aggregation
- */
- String PROP_EXCLUDE_FROM_AGGREGATE = "excludeFromAggregation";
-
- /**
- * String property: the function to index, for function-based index
- */
- String PROP_FUNCTION = "function";
-
- /**
- * Boolean property which signal LuceneIndexEditor to refresh the stored index definition
- */
- String PROP_REFRESH_DEFN = "refresh";
-
- /**
- * Boolean property to indicate that nodes nodetype matching indexRule name
- * should be indexed
- */
- String PROP_INDEX_NODE_TYPE = "nodeTypeIndex";
}
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexDefinition.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexDefinition.java?rev=1841926&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexDefinition.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexDefinition.java Tue Sep 25 12:24:15 2018
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import java.util.Collections;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.TokenizerChain;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.CommitMitigatingTieredMergePolicy;
+import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexFormatVersion;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.miscellaneous.LimitTokenCountAnalyzer;
+import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.analysis.path.PathHierarchyTokenizerFactory;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.LogByteSizeMergePolicy;
+import org.apache.lucene.index.LogDocMergePolicy;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.google.common.collect.Maps.newHashMap;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.ANL_DEFAULT;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INDEX_ORIGINAL_TERM;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.VERSION;
+import static org.apache.jackrabbit.oak.plugins.index.search.util.ConfigUtil.getOptionalValue;
+
+public class LuceneIndexDefinition extends IndexDefinition {
+ private static final Logger log = LoggerFactory.getLogger(IndexDefinition.class);
+
+ private final boolean saveDirListing;
+
+ private final Map<String, Analyzer> analyzers;
+ private final Analyzer analyzer;
+
+ private final Codec codec;
+
+ private final int maxFieldLength;
+
+ public LuceneIndexDefinition(NodeState root, NodeState defn, String indexPath) {
+ this(root, getIndexDefinitionState(defn), determineIndexFormatVersion(defn), determineUniqueId(defn), indexPath);
+ }
+
+ LuceneIndexDefinition(NodeState root, NodeState defn, IndexFormatVersion version, String uid, String indexPath) {
+ super(root, defn, version, uid, indexPath);
+
+ this.saveDirListing = getOptionalValue(defn, LuceneIndexConstants.SAVE_DIR_LISTING, true);
+ this.maxFieldLength = getOptionalValue(defn, LuceneIndexConstants.MAX_FIELD_LENGTH, DEFAULT_MAX_FIELD_LENGTH);
+ this.analyzers = collectAnalyzers(defn);
+ this.analyzer = createAnalyzer();
+ this.codec = createCodec();
+ }
+
+ public static Builder newBuilder(NodeState root, NodeState defn, String indexPath){
+ return (Builder)new Builder()
+ .root(root)
+ .defn(defn)
+ .indexPath(indexPath);
+ }
+
+ public static class Builder extends IndexDefinition.Builder {
+ @Override
+ public LuceneIndexDefinition build() {
+ return (LuceneIndexDefinition)super.build();
+ }
+
+ @Override
+ public LuceneIndexDefinition.Builder reindex() {
+ super.reindex();
+ return this;
+ }
+
+ @Override
+ protected IndexDefinition createInstance(NodeState indexDefnStateToUse) {
+ return new LuceneIndexDefinition(root, indexDefnStateToUse, version, uid, indexPath);
+ }
+ }
+
+ @Override
+ protected String getDefaultFunctionName() {
+ return "lucene";//TODO should this be LuceneIndexConstants.TYPE_LUCENE?
+ }
+
+ @Override
+ protected double getDefaultCostPerEntry(IndexFormatVersion version) {
+ //For older format cost per entry would be higher as it does a runtime
+ //aggregation
+ return version == IndexFormatVersion.V1 ? 1.5 : 1.0;
+ }
+
+ public boolean saveDirListing() {
+ return saveDirListing;
+ }
+
+ @Nullable
+ public Codec getCodec() {
+ return codec;
+ }
+
+ @NotNull
+ public MergePolicy getMergePolicy() {
+ // MP is not cached to avoid complaining about multiple IWs with multiplexing writers
+ return createMergePolicy();
+ }
+
+ public Analyzer getAnalyzer(){
+ return analyzer;
+ }
+
+ //~---------------------------------------------------< Analyzer >
+
+ private Analyzer createAnalyzer() {
+ Analyzer result;
+ Analyzer defaultAnalyzer = LuceneIndexConstants.ANALYZER;
+ if (analyzers.containsKey(ANL_DEFAULT)){
+ defaultAnalyzer = analyzers.get(ANL_DEFAULT);
+ }
+ if (!evaluatePathRestrictions()){
+ result = defaultAnalyzer;
+ } else {
+ Map<String, Analyzer> analyzerMap = ImmutableMap.<String, Analyzer>builder()
+ .put(FieldNames.ANCESTORS,
+ new TokenizerChain(new PathHierarchyTokenizerFactory(Collections.emptyMap())))
+ .build();
+ result = new PerFieldAnalyzerWrapper(defaultAnalyzer, analyzerMap);
+ }
+
+ //In case of negative value no limits would be applied
+ if (maxFieldLength < 0){
+ return result;
+ }
+ return new LimitTokenCountAnalyzer(result, maxFieldLength);
+ }
+
+ private static Map<String, Analyzer> collectAnalyzers(NodeState defn) {
+ Map<String, Analyzer> analyzerMap = newHashMap();
+ NodeStateAnalyzerFactory factory = new NodeStateAnalyzerFactory(VERSION);
+ NodeState analyzersTree = defn.getChildNode(LuceneIndexConstants.ANALYZERS);
+ for (ChildNodeEntry cne : analyzersTree.getChildNodeEntries()) {
+ Analyzer a = factory.createInstance(cne.getNodeState());
+ analyzerMap.put(cne.getName(), a);
+ }
+
+ if (getOptionalValue(analyzersTree, INDEX_ORIGINAL_TERM, false) && !analyzerMap.containsKey(ANL_DEFAULT)) {
+ analyzerMap.put(ANL_DEFAULT, new OakAnalyzer(VERSION, true));
+ }
+
+ return ImmutableMap.copyOf(analyzerMap);
+ }
+
+
+ //~---------------------------------------------< utility >
+
+ private Codec createCodec() {
+ String codecName = getOptionalValue(definition, LuceneIndexConstants.CODEC_NAME, null);
+ Codec codec = null;
+ if (codecName != null) {
+ // prevent LUCENE-6482
+ // (also done in LuceneIndexProviderService, just to be save)
+ OakCodec ensureLucene46CodecLoaded = new OakCodec();
+ // to ensure the JVM doesn't optimize away object creation
+ // (probably not really needed; just to be save)
+ log.debug("Lucene46Codec is loaded: {}", ensureLucene46CodecLoaded);
+ codec = Codec.forName(codecName);
+ log.debug("Codec is loaded: {}", codecName);
+ } else if (fullTextEnabled) {
+ codec = new OakCodec();
+ }
+ return codec;
+ }
+
+ private MergePolicy createMergePolicy() {
+ String mmp = System.getProperty("oak.lucene.cmmp");
+ if (mmp != null) {
+ return new CommitMitigatingTieredMergePolicy();
+ }
+
+ String mergePolicyName = getOptionalValue(definition, LuceneIndexConstants.MERGE_POLICY_NAME, null);
+ MergePolicy mergePolicy = null;
+ if (mergePolicyName != null) {
+ if (mergePolicyName.equalsIgnoreCase("no")) {
+ mergePolicy = NoMergePolicy.COMPOUND_FILES;
+ } else if (mergePolicyName.equalsIgnoreCase("mitigated")) {
+ mergePolicy = new CommitMitigatingTieredMergePolicy();
+ } else if (mergePolicyName.equalsIgnoreCase("tiered") || mergePolicyName.equalsIgnoreCase("default")) {
+ mergePolicy = new TieredMergePolicy();
+ } else if (mergePolicyName.equalsIgnoreCase("logbyte")) {
+ mergePolicy = new LogByteSizeMergePolicy();
+ } else if (mergePolicyName.equalsIgnoreCase("logdoc")) {
+ mergePolicy = new LogDocMergePolicy();
+ }
+ }
+ if (mergePolicy == null) {
+ mergePolicy = new TieredMergePolicy();
+ }
+ return mergePolicy;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexDefinition.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java?rev=1841926&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java Tue Sep 25 12:24:15 2018
@@ -0,0 +1,404 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.index.IndexEditor;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
+import org.apache.jackrabbit.oak.plugins.index.search.Aggregate;
+import org.apache.jackrabbit.oak.plugins.index.search.Aggregate.Matcher;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.PropertyUpdateCallback;
+import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
+import org.apache.jackrabbit.oak.spi.commit.Editor;
+import org.apache.jackrabbit.oak.spi.filter.PathFilter;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.lucene.document.Document;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.jackrabbit.oak.commons.PathUtils.concat;
+
+/**
+ * {@link IndexEditor} implementation that is responsible for keeping the
+ * {@link LuceneIndex} up to date
+ *
+ * @see LuceneIndex
+ */
+public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot {
+
+ private static final Logger log =
+ LoggerFactory.getLogger(LuceneIndexEditor.class);
+
+ private final LuceneIndexEditorContext context;
+
+ /** Name of this node, or {@code null} for the root node. */
+ private final String name;
+
+ /** Parent editor or {@code null} if this is the root editor. */
+ private final LuceneIndexEditor parent;
+
+ /** Path of this editor, built lazily in {@link #getPath()}. */
+ private String path;
+
+ private boolean propertiesChanged = false;
+
+ private List<PropertyState> propertiesModified = Lists.newArrayList();
+
+ /**
+ * Flag indicating if the current tree being traversed has a deleted parent.
+ */
+ private final boolean isDeleted;
+
+ private IndexDefinition.IndexingRule indexingRule;
+
+ private List<Matcher> currentMatchers = Collections.emptyList();
+
+ private final MatcherState matcherState;
+
+ private final PathFilter.Result pathFilterResult;
+
+ LuceneIndexEditor(LuceneIndexEditorContext context) throws CommitFailedException {
+ this.parent = null;
+ this.name = null;
+ this.path = "/";
+ this.context = context;
+ this.isDeleted = false;
+ this.matcherState = MatcherState.NONE;
+ this.pathFilterResult = context.getDefinition().getPathFilter().filter(PathUtils.ROOT_PATH);
+ }
+
+ private LuceneIndexEditor(LuceneIndexEditor parent, String name,
+ MatcherState matcherState,
+ PathFilter.Result pathFilterResult,
+ boolean isDeleted) {
+ this.parent = parent;
+ this.name = name;
+ this.path = null;
+ this.context = parent.context;
+ this.isDeleted = isDeleted;
+ this.matcherState = matcherState;
+ this.pathFilterResult = pathFilterResult;
+ }
+
+ public String getPath() {
+ if (path == null) { // => parent != null
+ path = concat(parent.getPath(), name);
+ }
+ return path;
+ }
+
+ @Override
+ public void enter(NodeState before, NodeState after)
+ throws CommitFailedException {
+ if (EmptyNodeState.MISSING_NODE == before && parent == null){
+ context.enableReindexMode();
+ }
+
+ //Only check for indexing if the result is include.
+ //In case like TRAVERSE nothing needs to be indexed for those
+ //path
+ if (pathFilterResult == PathFilter.Result.INCLUDE) {
+ //For traversal in deleted sub tree before state has to be used
+ NodeState current = after.exists() ? after : before;
+ indexingRule = getDefinition().getApplicableIndexingRule(current);
+
+ if (indexingRule != null) {
+ currentMatchers = indexingRule.getAggregate().createMatchers(this);
+ }
+ }
+ }
+
+ @Override
+ public void leave(NodeState before, NodeState after)
+ throws CommitFailedException {
+ if (propertiesChanged || !before.exists()) {
+ String path = getPath();
+ if (addOrUpdate(path, after, before.exists())) {
+ long indexed = context.incIndexedNodes();
+ if (indexed % 1000 == 0) {
+ log.debug("[{}] => Indexed {} nodes...", getIndexName(), indexed);
+ }
+ }
+ }
+
+ for (Matcher m : matcherState.affectedMatchers){
+ m.markRootDirty();
+ }
+
+ if (parent == null) {
+ PropertyUpdateCallback callback = context.getPropertyUpdateCallback();
+ if (callback != null) {
+ callback.done();
+ }
+
+ try {
+ context.closeWriter();
+ } catch (IOException e) {
+ CommitFailedException ce = new CommitFailedException("Lucene", 4,
+ "Failed to close the Lucene index " + context.getIndexingContext().getIndexPath(), e);
+ context.getIndexingContext().indexUpdateFailed(ce);
+ throw ce;
+ }
+ if (context.getIndexedNodes() > 0) {
+ log.debug("[{}] => Indexed {} nodes, done.", getIndexName(), context.getIndexedNodes());
+ }
+ }
+ }
+
+ @Override
+ public void propertyAdded(PropertyState after) {
+ markPropertyChanged(after.getName());
+ checkAggregates(after.getName());
+ propertyUpdated(null, after);
+ }
+
+ @Override
+ public void propertyChanged(PropertyState before, PropertyState after) {
+ markPropertyChanged(before.getName());
+ propertiesModified.add(before);
+ checkAggregates(before.getName());
+ propertyUpdated(before, after);
+ }
+
+ @Override
+ public void propertyDeleted(PropertyState before) {
+ markPropertyChanged(before.getName());
+ propertiesModified.add(before);
+ checkAggregates(before.getName());
+ propertyUpdated(before, null);
+ }
+
+ @Override
+ public Editor childNodeAdded(String name, NodeState after) {
+ PathFilter.Result filterResult = getPathFilterResult(name);
+ if (filterResult != PathFilter.Result.EXCLUDE) {
+ return new LuceneIndexEditor(this, name, getMatcherState(name, after), filterResult, false);
+ }
+ return null;
+ }
+
+ @Override
+ public Editor childNodeChanged(
+ String name, NodeState before, NodeState after) {
+ PathFilter.Result filterResult = getPathFilterResult(name);
+ if (filterResult != PathFilter.Result.EXCLUDE) {
+ return new LuceneIndexEditor(this, name, getMatcherState(name, after), filterResult, false);
+ }
+ return null;
+ }
+
+ @Override
+ public Editor childNodeDeleted(String name, NodeState before)
+ throws CommitFailedException {
+ PathFilter.Result filterResult = getPathFilterResult(name);
+ if (filterResult == PathFilter.Result.EXCLUDE) {
+ return null;
+ }
+
+ if (!isDeleted) {
+ // tree deletion is handled on the parent node
+ String path = concat(getPath(), name);
+ try {
+ LuceneIndexWriter writer = context.getWriter();
+ // Remove all index entries in the removed subtree
+ writer.deleteDocuments(path);
+ this.context.indexUpdate();
+ } catch (IOException e) {
+ CommitFailedException ce = new CommitFailedException("Lucene", 5, "Failed to remove the index entries of"
+ + " the removed subtree " + path + "for index " + context.getIndexingContext().getIndexPath(), e);
+ context.getIndexingContext().indexUpdateFailed(ce);
+ throw ce;
+ }
+ }
+
+ MatcherState ms = getMatcherState(name, before);
+ if (!ms.isEmpty()){
+ return new LuceneIndexEditor(this, name, ms, filterResult, true);
+ }
+ return null; // no need to recurse down the removed subtree
+ }
+
+ LuceneIndexEditorContext getContext() {
+ return context;
+ }
+
+ private boolean addOrUpdate(String path, NodeState state, boolean isUpdate)
+ throws CommitFailedException {
+ try {
+ Document d = makeDocument(path, state, isUpdate);
+ if (d != null) {
+ if (log.isTraceEnabled()) {
+ log.trace("[{}] Indexed document for {} is {}", getIndexName(), path, d);
+ }
+ context.indexUpdate();
+ context.getWriter().updateDocument(path, d);
+ return true;
+ }
+ } catch (IOException e) {
+ CommitFailedException ce = new CommitFailedException("Lucene", 3,
+ "Failed to index the node " + path, e);
+ context.getIndexingContext().indexUpdateFailed(ce);
+ throw ce;
+ } catch (IllegalArgumentException ie) {
+ log.warn("Failed to index the node [{}]", path, ie);
+ }
+ return false;
+ }
+
+ private Document makeDocument(String path, NodeState state, boolean isUpdate) throws IOException {
+ if (!isIndexable()) {
+ return null;
+ }
+ return context.newDocumentMaker(indexingRule, path).makeDocument(state, isUpdate, propertiesModified);
+ }
+
+
+ //~-------------------------------------------------------< Aggregate >
+
+ @Override
+ public void markDirty() {
+ propertiesChanged = true;
+ }
+
+ private MatcherState getMatcherState(String name, NodeState after) {
+ List<Matcher> matched = Lists.newArrayList();
+ List<Matcher> inherited = Lists.newArrayList();
+ for (Matcher m : Iterables.concat(matcherState.inherited, currentMatchers)) {
+ Matcher result = m.match(name, after);
+ if (result.getStatus() == Matcher.Status.MATCH_FOUND){
+ matched.add(result);
+ }
+
+ if (result.getStatus() != Matcher.Status.FAIL){
+ inherited.addAll(result.nextSet());
+ }
+ }
+
+ if (!matched.isEmpty() || !inherited.isEmpty()) {
+ return new MatcherState(matched, inherited);
+ }
+ return MatcherState.NONE;
+ }
+
+
+ /**
+ * Determines which all matchers are affected by this property change
+ *
+ * @param name modified property name
+ */
+ private void checkAggregates(String name) {
+ for (Matcher m : matcherState.matched) {
+ if (!matcherState.affectedMatchers.contains(m)
+ && m.aggregatesProperty(name)) {
+ matcherState.affectedMatchers.add(m);
+ }
+ }
+ }
+
+ private static class MatcherState {
+ final static MatcherState NONE = new MatcherState(Collections.<Matcher>emptyList(),
+ Collections.<Matcher>emptyList());
+
+ final List<Matcher> matched;
+ final List<Matcher> inherited;
+ final Set<Matcher> affectedMatchers;
+
+ public MatcherState(List<Matcher> matched,
+ List<Matcher> inherited){
+ this.matched = matched;
+ this.inherited = inherited;
+
+ //Affected matches would only be used when there are
+ //some matched matchers
+ if (matched.isEmpty()){
+ affectedMatchers = Collections.emptySet();
+ } else {
+ affectedMatchers = Sets.newIdentityHashSet();
+ }
+ }
+
+ public boolean isEmpty() {
+ return matched.isEmpty() && inherited.isEmpty();
+ }
+ }
+
+ private void markPropertyChanged(String name) {
+ if (isIndexable()
+ && !propertiesChanged
+ && indexingRule.isIndexed(name)) {
+ propertiesChanged = true;
+ }
+ }
+
+ private void propertyUpdated(PropertyState before, PropertyState after) {
+ PropertyUpdateCallback callback = context.getPropertyUpdateCallback();
+
+ //Avoid further work if no callback is present
+ if (callback == null) {
+ return;
+ }
+
+ String propertyName = before != null ? before.getName() : after.getName();
+
+ if (isIndexable()) {
+ PropertyDefinition pd = indexingRule.getConfig(propertyName);
+ if (pd != null) {
+ callback.propertyUpdated(getPath(), propertyName, pd, before, after);
+ }
+ }
+
+ for (Matcher m : matcherState.matched) {
+ if (m.aggregatesProperty(propertyName)) {
+ Aggregate.Include i = m.getCurrentInclude();
+ if (i instanceof Aggregate.PropertyInclude) {
+ PropertyDefinition pd = ((Aggregate.PropertyInclude) i).getPropertyDefinition();
+ String propertyRelativePath = PathUtils.concat(m.getMatchedPath(), propertyName);
+
+ callback.propertyUpdated(m.getRootPath(), propertyRelativePath, pd, before, after);
+ }
+ }
+ }
+ }
+
+ private IndexDefinition getDefinition() {
+ return context.getDefinition();
+ }
+
+ private boolean isIndexable(){
+ return indexingRule != null;
+ }
+
+ private PathFilter.Result getPathFilterResult(String childNodeName) {
+ return context.getDefinition().getPathFilter().filter(concat(getPath(), childNodeName));
+ }
+
+ private String getIndexName() {
+ return context.getDefinition().getIndexName();
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java?rev=1841926&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java Tue Sep 25 12:24:15 2018
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import java.io.IOException;
+
+import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback;
+import org.apache.jackrabbit.oak.plugins.index.IndexingContext;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.FacetHelper;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.FacetsConfigProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
+import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexEditorContext;
+import org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexWriterFactory;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.apache.lucene.facet.FacetsConfig;
+import org.jetbrains.annotations.Nullable;
+
+public class LuceneIndexEditorContext extends FulltextIndexEditorContext implements FacetsConfigProvider {
+ private FacetsConfig facetsConfig;
+
+ private final IndexAugmentorFactory augmentorFactory;
+
+ LuceneIndexEditorContext(NodeState root, NodeBuilder definition,
+ @Nullable IndexDefinition indexDefinition,
+ IndexUpdateCallback updateCallback,
+ FulltextIndexWriterFactory indexWriterFactory,
+ ExtractedTextCache extractedTextCache,
+ IndexAugmentorFactory augmentorFactory,
+ IndexingContext indexingContext, boolean asyncIndexing) {
+ super(root, definition, indexDefinition, updateCallback, indexWriterFactory, extractedTextCache,
+ indexingContext, asyncIndexing);
+ this.augmentorFactory = augmentorFactory;
+ }
+
+ @Override
+ public IndexDefinition.Builder newDefinitionBuilder() {
+ return new LuceneIndexDefinition.Builder();
+ }
+
+ @Override
+ public LuceneDocumentMaker newDocumentMaker(IndexDefinition.IndexingRule rule, String path){
+ //Faceting is only enabled for async mode
+ FacetsConfigProvider facetsConfigProvider = isAsyncIndexing() ? this : null;
+ return new LuceneDocumentMaker(getTextExtractor(), facetsConfigProvider, augmentorFactory,
+ definition, rule, path);
+ }
+
+ @Override
+ public LuceneIndexWriter getWriter() throws IOException {
+ return (LuceneIndexWriter)super.getWriter();
+ }
+
+ @Override
+ public FacetsConfig getFacetsConfig() {
+ if (facetsConfig == null){
+ facetsConfig = FacetHelper.getFacetsConfig(definitionBuilder);
+ }
+ return facetsConfig;
+ }
+
+ /** Only set for testing */
+ public static void setClock(Clock c) {
+ FulltextIndexEditorContext.setClock(c);
+ }
+}
\ No newline at end of file
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java?rev=1841926&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java Tue Sep 25 12:24:15 2018
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.plugins.index.ContextAwareCallback;
+import org.apache.jackrabbit.oak.plugins.index.IndexEditor;
+import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback;
+import org.apache.jackrabbit.oak.plugins.index.IndexingContext;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory.ActiveDeletedBlobCollector;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory.BlobDeletionCallback;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.DefaultDirectoryFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.IndexingQueue;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexWriterFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LuceneDocumentHolder;
+import org.apache.jackrabbit.oak.plugins.index.lucene.property.LuceneIndexPropertyQuery;
+import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexUpdateCallback;
+import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyQuery;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.DefaultIndexWriterFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterConfig;
+import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexWriterFactory;
+import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitContext;
+import org.apache.jackrabbit.oak.spi.commit.Editor;
+import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
+import org.apache.jackrabbit.oak.spi.mount.Mounts;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.ReadOnlyBuilder;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE;
+
+/**
+ * Service that provides Lucene based {@link IndexEditor}s
+ *
+ * @see LuceneIndexEditor
+ * @see IndexEditorProvider
+ *
+ */
+public class LuceneIndexEditorProvider implements IndexEditorProvider {
+ private final Logger log = LoggerFactory.getLogger(getClass());
+ private final IndexCopier indexCopier;
+ private final ExtractedTextCache extractedTextCache;
+ private final IndexAugmentorFactory augmentorFactory;
+ private final IndexTracker indexTracker;
+ private final MountInfoProvider mountInfoProvider;
+ private final ActiveDeletedBlobCollector activeDeletedBlobCollector;
+ private GarbageCollectableBlobStore blobStore;
+ private IndexingQueue indexingQueue;
+ private boolean nrtIndexingEnabled;
+ private LuceneIndexWriterConfig writerConfig = new LuceneIndexWriterConfig();
+
+ /**
+ * Number of indexed Lucene document that can be held in memory
+ * This ensures that for very large commit memory consumption
+ * is bounded
+ */
+ private int inMemoryDocsLimit = Integer.getInteger("oak.lucene.inMemoryDocsLimit", 500);
+
+ public LuceneIndexEditorProvider() {
+ this(null);
+ }
+
+ public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier) {
+ //Disable the cache by default in ExtractedTextCache
+ this(indexCopier, new ExtractedTextCache(0, 0));
+ }
+
+ public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier,
+ ExtractedTextCache extractedTextCache) {
+ this(indexCopier, extractedTextCache, null, Mounts.defaultMountInfoProvider());
+ }
+
+ public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier,
+ ExtractedTextCache extractedTextCache,
+ @Nullable IndexAugmentorFactory augmentorFactory,
+ MountInfoProvider mountInfoProvider) {
+ this(indexCopier, null, extractedTextCache, augmentorFactory, mountInfoProvider);
+ }
+
+ public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier,
+ @Nullable IndexTracker indexTracker,
+ ExtractedTextCache extractedTextCache,
+ @Nullable IndexAugmentorFactory augmentorFactory,
+ MountInfoProvider mountInfoProvider) {
+ this(indexCopier, indexTracker, extractedTextCache, augmentorFactory, mountInfoProvider,
+ ActiveDeletedBlobCollectorFactory.NOOP);
+ }
+ public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier,
+ @Nullable IndexTracker indexTracker,
+ ExtractedTextCache extractedTextCache,
+ @Nullable IndexAugmentorFactory augmentorFactory,
+ MountInfoProvider mountInfoProvider,
+ @NotNull ActiveDeletedBlobCollectorFactory.ActiveDeletedBlobCollector activeDeletedBlobCollector) {
+ this.indexCopier = indexCopier;
+ this.indexTracker = indexTracker;
+ this.extractedTextCache = extractedTextCache != null ? extractedTextCache : new ExtractedTextCache(0, 0);
+ this.augmentorFactory = augmentorFactory;
+ this.mountInfoProvider = checkNotNull(mountInfoProvider);
+ this.activeDeletedBlobCollector = activeDeletedBlobCollector;
+ }
+
+ @Override
+ public Editor getIndexEditor(
+ @NotNull String type, @NotNull NodeBuilder definition, @NotNull NodeState root,
+ @NotNull IndexUpdateCallback callback)
+ throws CommitFailedException {
+ if (TYPE_LUCENE.equals(type)) {
+ checkArgument(callback instanceof ContextAwareCallback, "callback instance not of type " +
+ "ContextAwareCallback [%s]", callback);
+ IndexingContext indexingContext = ((ContextAwareCallback)callback).getIndexingContext();
+ BlobDeletionCallback blobDeletionCallback = activeDeletedBlobCollector.getBlobDeletionCallback();
+ indexingContext.registerIndexCommitCallback(blobDeletionCallback);
+ FulltextIndexWriterFactory writerFactory = null;
+ LuceneIndexDefinition indexDefinition = null;
+ boolean asyncIndexing = true;
+ String indexPath = indexingContext.getIndexPath();
+ PropertyIndexUpdateCallback propertyUpdateCallback = null;
+
+ if (nrtIndexingEnabled() && !indexingContext.isAsync() && IndexDefinition.supportsSyncOrNRTIndexing(definition)) {
+
+ //Would not participate in reindexing. Only interested in
+ //incremental indexing
+ if (indexingContext.isReindexing()){
+ return null;
+ }
+
+ CommitContext commitContext = getCommitContext(indexingContext);
+ if (commitContext == null){
+ //Logically there should not be any commit without commit context. But
+ //some initializer code does the commit with out it. So ignore such calls with
+ //warning now
+ //TODO Revisit use of warn level once all such cases are analyzed
+ log.warn("No CommitContext found for commit", new Exception());
+ return null;
+ }
+
+ //TODO Also check if index has been done once
+
+
+ writerFactory = new LocalIndexWriterFactory(getDocumentHolder(commitContext),
+ indexPath);
+
+ //IndexDefinition from tracker might differ from one passed here for reindexing
+ //case which should be fine. However reusing existing definition would avoid
+ //creating definition instance for each commit as this gets executed for each commit
+ if (indexTracker != null){
+ indexDefinition = indexTracker.getIndexDefinition(indexPath);
+ if (indexDefinition != null && !indexDefinition.hasMatchingNodeTypeReg(root)){
+ log.debug("Detected change in NodeType registry for index {}. Would not use " +
+ "existing index definition", indexDefinition.getIndexPath());
+ indexDefinition = null;
+ }
+ }
+
+ if (indexDefinition == null) {
+ indexDefinition = LuceneIndexDefinition.newBuilder(root, definition.getNodeState(),
+ indexPath).build();
+ }
+
+ if (indexDefinition.hasSyncPropertyDefinitions()) {
+ propertyUpdateCallback = new PropertyIndexUpdateCallback(indexPath, definition, root);
+ if (indexTracker != null) {
+ PropertyQuery query = new LuceneIndexPropertyQuery(indexTracker, indexPath);
+ propertyUpdateCallback.getUniquenessConstraintValidator().setSecondStore(query);
+ }
+ }
+
+ //Pass on a read only builder to ensure that nothing gets written
+ //at all to NodeStore for local indexing.
+ //TODO [hybrid] This would cause issue with Facets as for faceted fields
+ //some stuff gets written to NodeBuilder. That logic should be refactored
+ //to be moved to LuceneIndexWriter
+ definition = new ReadOnlyBuilder(definition.getNodeState());
+
+ asyncIndexing = false;
+ }
+
+ if (writerFactory == null) {
+ writerFactory = new DefaultIndexWriterFactory(mountInfoProvider, newDirectoryFactory(blobDeletionCallback), writerConfig);
+ }
+
+ LuceneIndexEditorContext context = new LuceneIndexEditorContext(root, definition, indexDefinition, callback,
+ writerFactory, extractedTextCache, augmentorFactory, indexingContext, asyncIndexing);
+
+ context.setPropertyUpdateCallback(propertyUpdateCallback);
+ return new LuceneIndexEditor(context);
+ }
+ return null;
+ }
+
+ IndexCopier getIndexCopier() {
+ return indexCopier;
+ }
+
+ IndexingQueue getIndexingQueue() {
+ return indexingQueue;
+ }
+
+ public ExtractedTextCache getExtractedTextCache() {
+ return extractedTextCache;
+ }
+
+ public void setInMemoryDocsLimit(int inMemoryDocsLimit) {
+ this.inMemoryDocsLimit = inMemoryDocsLimit;
+ }
+
+ protected DirectoryFactory newDirectoryFactory(BlobDeletionCallback blobDeletionCallback) {
+ return new DefaultDirectoryFactory(indexCopier, blobStore, blobDeletionCallback);
+ }
+
+ private LuceneDocumentHolder getDocumentHolder(CommitContext commitContext){
+ LuceneDocumentHolder holder = (LuceneDocumentHolder) commitContext.get(LuceneDocumentHolder.NAME);
+ if (holder == null) {
+ holder = new LuceneDocumentHolder(indexingQueue, inMemoryDocsLimit);
+ commitContext.set(LuceneDocumentHolder.NAME, holder);
+ }
+ return holder;
+ }
+
+ public void setBlobStore(@Nullable GarbageCollectableBlobStore blobStore) {
+ this.blobStore = blobStore;
+ }
+
+ public void setIndexingQueue(IndexingQueue indexingQueue) {
+ this.indexingQueue = indexingQueue;
+ this.nrtIndexingEnabled = indexingQueue != null;
+ }
+
+ public void setWriterConfig(LuceneIndexWriterConfig writerConfig) {
+ this.writerConfig = writerConfig;
+ }
+
+ GarbageCollectableBlobStore getBlobStore() {
+ return blobStore;
+ }
+
+ private boolean nrtIndexingEnabled() {
+ return nrtIndexingEnabled;
+ }
+
+ private static CommitContext getCommitContext(IndexingContext indexingContext) {
+ return (CommitContext) indexingContext.getCommitInfo().getInfo().get(CommitContext.NAME);
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java?rev=1841926&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java Tue Sep 25 12:24:15 2018
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.json.JsopDiff;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfo;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.IndexInfo;
+import org.apache.jackrabbit.oak.plugins.index.IndexInfoProvider;
+import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryUtils;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.OakDirectory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.MultiplexersLucene;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.util.NodeStateCloner;
+import org.apache.jackrabbit.oak.spi.state.EqualsDiff;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.spi.state.ReadOnlyBuilder;
+import org.apache.jackrabbit.util.ISO8601;
+import org.apache.lucene.store.Directory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.INDEX_DEFINITION_NODE;
+
+public class LuceneIndexInfoProvider implements IndexInfoProvider {
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ private final NodeStore nodeStore;
+
+ private final AsyncIndexInfoService asyncInfoService;
+
+ private final File workDir;
+
+ public LuceneIndexInfoProvider(NodeStore nodeStore, AsyncIndexInfoService asyncInfoService, File workDir) {
+ this.nodeStore = checkNotNull(nodeStore);
+ this.asyncInfoService = checkNotNull(asyncInfoService);
+ this.workDir = checkNotNull(workDir);
+ }
+
+ @Override
+ public String getType() {
+ return LuceneIndexConstants.TYPE_LUCENE;
+ }
+
+ @Override
+ public IndexInfo getInfo(String indexPath) throws IOException {
+ NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(), indexPath);
+
+ checkArgument(LuceneIndexConstants.TYPE_LUCENE.equals(idxState.getString(IndexConstants.TYPE_PROPERTY_NAME)),
+ "Index definition at [%s] is not of type 'lucene'", indexPath);
+
+ LuceneIndexInfo info = new LuceneIndexInfo(indexPath);
+ computeSize(idxState, info);
+ computeIndexDefinitionChange(idxState, info);
+ computeLastUpdatedTime(idxState, info);
+ computeAsyncIndexInfo(idxState, indexPath, info);
+ return info;
+ }
+
+ @Override
+ public boolean isValid(String indexPath) throws IOException {
+ IndexConsistencyChecker checker = new IndexConsistencyChecker(nodeStore.getRoot(), indexPath, workDir);
+
+ boolean result = false;
+ try{
+ result = checker.check(IndexConsistencyChecker.Level.BLOBS_ONLY).clean;
+ } catch (Exception e) {
+ log.warn("Error occurred while performing consistency check for {}", indexPath, e);
+ }
+ return result;
+ }
+
+ private void computeAsyncIndexInfo(NodeState idxState, String indexPath, LuceneIndexInfo info) {
+ String asyncName = IndexUtils.getAsyncLaneName(idxState, indexPath);
+ if (asyncName == null) {
+ log.warn("No 'async' value for index definition at [{}]. Definition {}", indexPath, idxState);
+ return;
+ }
+
+ AsyncIndexInfo asyncInfo = asyncInfoService.getInfo(asyncName);
+ checkNotNull(asyncInfo, "No async info found for name [%s] " +
+ "for index at [%s]", asyncName, indexPath);
+
+ info.indexedUptoTime = asyncInfo.getLastIndexedTo();
+ info.asyncName = asyncName;
+ }
+
+ private void computeSize(NodeState idxState, LuceneIndexInfo info) throws IOException {
+ LuceneIndexDefinition defn = LuceneIndexDefinition.newBuilder(nodeStore.getRoot(), idxState, info.indexPath).build();
+ for (String dirName : idxState.getChildNodeNames()) {
+ if (NodeStateUtils.isHidden(dirName) && MultiplexersLucene.isIndexDirName(dirName)) {
+ try (Directory dir = new OakDirectory(new ReadOnlyBuilder(idxState), dirName, defn, true)) {
+ info.numEntries += DirectoryUtils.getNumDocs(dir);
+ info.size = DirectoryUtils.dirSize(dir);
+ }
+ }
+ }
+ }
+
+ private static void computeLastUpdatedTime(NodeState idxState, LuceneIndexInfo info) {
+ NodeState status = idxState.getChildNode(IndexDefinition.STATUS_NODE);
+ if (status.exists()){
+ PropertyState updatedTime = status.getProperty(IndexDefinition.STATUS_LAST_UPDATED);
+ if (updatedTime != null) {
+ info.lastUpdatedTime = ISO8601.parse(updatedTime.getValue(Type.DATE)).getTimeInMillis();
+ }
+ }
+ }
+
+ private static void computeIndexDefinitionChange(NodeState idxState, LuceneIndexInfo info) {
+ NodeState storedDefn = idxState.getChildNode(INDEX_DEFINITION_NODE);
+ if (storedDefn.exists()) {
+ NodeState currentDefn = NodeStateCloner.cloneVisibleState(idxState);
+ if (!FilteringEqualsDiff.equals(storedDefn, currentDefn)){
+ info.indexDefinitionChanged = true;
+ info.indexDiff = JsopDiff.diffToJsop(storedDefn, currentDefn);
+ }
+ }
+ }
+
+ private static class LuceneIndexInfo implements IndexInfo {
+ String indexPath;
+ String asyncName;
+ long numEntries;
+ long size;
+ long indexedUptoTime;
+ long lastUpdatedTime;
+ boolean indexDefinitionChanged;
+ String indexDiff;
+
+ public LuceneIndexInfo(String indexPath) {
+ this.indexPath = indexPath;
+ }
+
+ @Override
+ public String getIndexPath() {
+ return indexPath;
+ }
+
+ @Override
+ public String getType() {
+ return LuceneIndexConstants.TYPE_LUCENE;
+ }
+
+ @Override
+ public String getAsyncLaneName() {
+ return asyncName;
+ }
+
+ @Override
+ public long getLastUpdatedTime() {
+ return lastUpdatedTime;
+ }
+
+ @Override
+ public long getIndexedUpToTime() {
+ return indexedUptoTime;
+ }
+
+ @Override
+ public long getEstimatedEntryCount() {
+ return numEntries;
+ }
+
+ @Override
+ public long getSizeInBytes() {
+ return size;
+ }
+
+ @Override
+ public boolean hasIndexDefinitionChangedWithoutReindexing() {
+ return indexDefinitionChanged;
+ }
+
+ @Override
+ public String getIndexDefinitionDiff() {
+ return indexDiff;
+ }
+ }
+
+ static class FilteringEqualsDiff extends EqualsDiff {
+ private static final Set<String> IGNORED_PROP_NAMES = ImmutableSet.of(
+ IndexConstants.REINDEX_COUNT,
+ IndexConstants.REINDEX_PROPERTY_NAME
+ );
+ public static boolean equals(NodeState before, NodeState after) {
+ return before.exists() == after.exists()
+ && after.compareAgainstBaseState(before, new FilteringEqualsDiff());
+ }
+
+ @Override
+ public boolean propertyChanged(PropertyState before, PropertyState after) {
+ if (ignoredProp(before.getName())){
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean propertyAdded(PropertyState after) {
+ if (ignoredProp(after.getName())){
+ return true;
+ }
+ return super.propertyAdded(after);
+ }
+
+ @Override
+ public boolean propertyDeleted(PropertyState before) {
+ if (ignoredProp(before.getName())){
+ return true;
+ }
+ return super.propertyDeleted(before);
+ }
+
+ private boolean ignoredProp(String name) {
+ return IGNORED_PROP_NAMES.contains(name) || NodeStateUtils.isHidden(name);
+ }
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java?rev=1841926&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java Tue Sep 25 12:24:15 2018
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import java.util.Collection;
+import java.util.function.Predicate;
+
+import org.apache.jackrabbit.oak.plugins.index.search.IndexFormatVersion;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexLookup;
+import org.apache.jackrabbit.oak.spi.query.Filter;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE;
+
+class LuceneIndexLookupUtil {
+ static final Predicate<NodeState> LUCENE_INDEX_DEFINITION_PREDICATE =
+ state -> TYPE_LUCENE.equals(state.getString(TYPE_PROPERTY_NAME));
+
+ private LuceneIndexLookupUtil() {
+ }
+
+ /**
+ * Returns the path of the first Lucene index node which supports
+ * fulltext search
+ */
+ public static String getOldFullTextIndexPath(NodeState root, Filter filter, IndexTracker tracker) {
+ Collection<String> indexPaths = getLuceneIndexLookup(root).collectIndexNodePaths(filter, false);
+ LuceneIndexNode indexNode = null;
+ for (String path : indexPaths) {
+ try {
+ indexNode = tracker.acquireIndexNode(path);
+ if (indexNode != null
+ && indexNode.getDefinition().isFullTextEnabled()
+ && indexNode.getDefinition().getVersion() == IndexFormatVersion.V1) {
+ return path;
+ }
+ } finally {
+ if (indexNode != null) {
+ indexNode.release();
+ }
+ }
+ }
+ return null;
+ }
+
+ public static IndexLookup getLuceneIndexLookup(NodeState root) {
+ return new IndexLookup(root, LUCENE_INDEX_DEFINITION_PREDICATE);
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java?rev=1841926&r1=1841925&r2=1841926&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java Tue Sep 25 12:24:15 2018
@@ -48,7 +48,7 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.json.JsopDiff;
import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
-import org.apache.jackrabbit.oak.plugins.index.lucene.BadIndexTracker.BadIndexInfo;
+import org.apache.jackrabbit.oak.plugins.index.search.BadIndexTracker.BadIndexInfo;
import org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexInfo;
import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.PathStoredFieldVisitor;
@@ -56,6 +56,12 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker.Level;
import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker.Result;
import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader;
+import org.apache.jackrabbit.oak.plugins.index.search.BadIndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
+import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexNode;
+import org.apache.jackrabbit.oak.plugins.index.search.util.NodeStateCloner;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
@@ -85,7 +91,7 @@ import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
-import static org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.INDEX_DEFINITION_NODE;
+import static org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.INDEX_DEFINITION_NODE;
import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newAncestorTerm;
import static org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryUtils.dirSize;
@@ -116,7 +122,7 @@ public class LuceneIndexMBeanImpl extend
tds = new TabularDataSupport(tt);
Set<String> indexes = indexTracker.getIndexNodePaths();
for (String path : indexes) {
- IndexNode indexNode = null;
+ LuceneIndexNode indexNode = null;
try {
indexNode = indexTracker.acquireIndexNode(path);
if (indexNode != null) {
@@ -144,7 +150,7 @@ public class LuceneIndexMBeanImpl extend
tds = new TabularDataSupport(tt);
Set<String> indexes = indexTracker.getBadIndexTracker().getIndexPaths();
for (String path : indexes) {
- BadIndexInfo info = indexTracker.getBadIndexTracker().getInfo(path);
+ BadIndexTracker.BadIndexInfo info = indexTracker.getBadIndexTracker().getInfo(path);
if (info != null){
BadIndexStats stats = new BadIndexStats(info);
tds.put(stats.toCompositeData());
@@ -184,7 +190,7 @@ public class LuceneIndexMBeanImpl extend
@Override
public String[] getIndexedPaths(String indexPath, int maxLevel, int maxPathCount) throws IOException {
- IndexNode indexNode = null;
+ LuceneIndexNode indexNode = null;
try {
if(indexPath == null){
indexPath = "/";
@@ -195,7 +201,7 @@ public class LuceneIndexMBeanImpl extend
IndexDefinition defn = indexNode.getDefinition();
if (!defn.evaluatePathRestrictions()){
String msg = String.format("Index at [%s] does not have [%s] enabled. So paths statistics cannot " +
- "be determined for this index", indexPath, LuceneIndexConstants.EVALUATE_PATH_RESTRICTION);
+ "be determined for this index", indexPath, FulltextIndexConstants.EVALUATE_PATH_RESTRICTION);
return createMsg(msg);
}
@@ -220,7 +226,7 @@ public class LuceneIndexMBeanImpl extend
}
ArrayList<String> list = new ArrayList<String>();
for (String path : indexes) {
- IndexNode indexNode = null;
+ LuceneIndexNode indexNode = null;
try {
indexNode = indexTracker.acquireIndexNode(path);
if (indexNode != null) {
@@ -255,7 +261,7 @@ public class LuceneIndexMBeanImpl extend
}
ArrayList<String> list = new ArrayList<String>();
for (String path : indexes) {
- IndexNode indexNode = null;
+ LuceneIndexNode indexNode = null;
try {
indexNode = indexTracker.acquireIndexNode(path);
if (indexNode != null) {
@@ -370,7 +376,7 @@ public class LuceneIndexMBeanImpl extend
}
public void dumpIndexContent(String sourcePath, String destPath) throws IOException {
- IndexNode indexNode = null;
+ LuceneIndexNode indexNode = null;
try {
if(sourcePath == null){
sourcePath = "/";
@@ -681,7 +687,7 @@ public class LuceneIndexMBeanImpl extend
private final String nrtIndexSizeStr;
private final int numDocsNRT;
- public IndexStats(String path, IndexNode indexNode) throws IOException {
+ public IndexStats(String path, LuceneIndexNode indexNode) throws IOException {
this.path = path;
numDocs = indexNode.getSearcher().getIndexReader().numDocs();
maxDoc = indexNode.getSearcher().getIndexReader().maxDoc();
@@ -755,9 +761,9 @@ public class LuceneIndexMBeanImpl extend
}
}
- private final BadIndexInfo info;
+ private final BadIndexTracker.BadIndexInfo info;
- public BadIndexStats(BadIndexInfo info){
+ public BadIndexStats(BadIndexTracker.BadIndexInfo info){
this.info = info;
}
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java?rev=1841926&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java Tue Sep 25 12:24:15 2018
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexNode;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester;
+import org.apache.lucene.store.Directory;
+import org.jetbrains.annotations.Nullable;
+
+public interface LuceneIndexNode extends IndexNode {
+
+ @Override
+ LuceneIndexDefinition getDefinition();
+
+ IndexSearcher getSearcher();
+
+ LuceneIndexStatistics getIndexStatistics();
+
+ List<LuceneIndexReader> getPrimaryReaders();
+
+ @Nullable
+ Directory getSuggestDirectory();
+
+ List<LuceneIndexReader> getNRTReaders();
+
+ @Nullable
+ AnalyzingInfixSuggester getLookup();
+
+ int getIndexNodeId();
+
+ @Nullable
+ LuceneIndexWriter getLocalWriter() throws IOException;
+
+ void refreshReadersOnWriteIfRequired();
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java
------------------------------------------------------------------------------
svn:eol-style = native