You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by al...@apache.org on 2014/04/16 22:20:09 UTC
svn commit: r1588058 - in /jackrabbit/oak/branches/1.0: ./
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/
oak-lucene/src/test/java/org/apache/jackra...
Author: alexparvulescu
Date: Wed Apr 16 20:20:09 2014
New Revision: 1588058
URL: http://svn.apache.org/r1588058
Log:
OAK-1738 Provide experimental config flag to enable/disable storing the content in the index
- merged to 1.0
Modified:
jackrabbit/oak/branches/1.0/ (props changed)
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneIndexHelper.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneInitializerHelper.java
jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java
Propchange: jackrabbit/oak/branches/1.0/
------------------------------------------------------------------------------
Merged /jackrabbit/oak/trunk:r1588046
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java?rev=1588058&r1=1588057&r2=1588058&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java Wed Apr 16 20:20:09 2014
@@ -20,7 +20,8 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
+
+import static org.apache.lucene.index.FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames.PATH;
import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames.FULLTEXT;
@@ -38,19 +39,28 @@ public final class FieldFactory {
*/
private static final FieldType OAK_TYPE = new FieldType();
+ private static final FieldType OAK_TYPE_NOT_STORED = new FieldType();
+
static {
OAK_TYPE.setIndexed(true);
OAK_TYPE.setOmitNorms(true);
OAK_TYPE.setStored(true);
- OAK_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ OAK_TYPE.setIndexOptions(DOCS_AND_FREQS_AND_POSITIONS);
OAK_TYPE.setTokenized(true);
OAK_TYPE.freeze();
+
+ OAK_TYPE_NOT_STORED.setIndexed(true);
+ OAK_TYPE_NOT_STORED.setOmitNorms(true);
+ OAK_TYPE_NOT_STORED.setStored(false);
+ OAK_TYPE_NOT_STORED.setIndexOptions(DOCS_AND_FREQS_AND_POSITIONS);
+ OAK_TYPE_NOT_STORED.setTokenized(true);
+ OAK_TYPE_NOT_STORED.freeze();
}
private final static class OakTextField extends Field {
- public OakTextField(String name, String value) {
- super(name, value, OAK_TYPE);
+ public OakTextField(String name, String value, boolean stored) {
+ super(name, value, stored ? OAK_TYPE : OAK_TYPE_NOT_STORED);
}
}
@@ -65,9 +75,9 @@ public final class FieldFactory {
}
public static Field newPropertyField(String name, String value,
- boolean tokenized) {
+ boolean tokenized, boolean stored) {
if (tokenized) {
- return new OakTextField(name, value);
+ return new OakTextField(name, value, stored);
}
return new StringField(name, value, NO);
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1588058&r1=1588057&r2=1588058&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java Wed Apr 16 20:20:09 2014
@@ -49,4 +49,9 @@ public interface LuceneIndexConstants {
String INDEX_DATA_CHILD_NAME_FS = "data";
+ /**
+ * Experimental flag to control storage behavior: 'null' or 'true' means the content is stored
+ */
+ String EXPERIMENTAL_STORAGE = "oak.experimental.storage";
+
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java?rev=1588058&r1=1588057&r2=1588058&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java Wed Apr 16 20:20:09 2014
@@ -203,7 +203,8 @@ public class LuceneIndexEditor implement
for (String value : property.getValue(Type.STRINGS)) {
this.context.indexUpdate();
document.add(newPropertyField(pname, value,
- !skipTokenization(pname)));
+ !skipTokenization(pname),
+ context.isStored(pname)));
document.add(newFulltextField(value));
dirty = true;
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java?rev=1588058&r1=1588057&r2=1588058&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java Wed Apr 16 20:20:09 2014
@@ -17,6 +17,7 @@
package org.apache.jackrabbit.oak.plugins.index.lucene;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXCLUDE_PROPERTY_NAMES;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXPERIMENTAL_STORAGE;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INDEX_DATA_CHILD_NAME;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PERSISTENCE_PATH;
@@ -105,6 +106,8 @@ public class LuceneIndexEditorContext {
private long indexedNodes;
+ private boolean storageEnabled = true;
+
private final IndexUpdateCallback updateCallback;
LuceneIndexEditorContext(NodeBuilder definition, Analyzer analyzer, IndexUpdateCallback updateCallback) {
@@ -131,6 +134,8 @@ public class LuceneIndexEditorContext {
} else {
excludes = ImmutableSet.of();
}
+ PropertyState storage = definition.getProperty(EXPERIMENTAL_STORAGE);
+ storageEnabled = storage == null || storage.getValue(Type.BOOLEAN);
this.indexedNodes = 0;
this.updateCallback = updateCallback;
}
@@ -176,4 +181,12 @@ public class LuceneIndexEditorContext {
updateCallback.indexUpdate();
}
+ /**
+ * Checks if a given property should be stored in the lucene index or not
+ *
+ */
+ public boolean isStored(String name) {
+ return storageEnabled;
+ }
+
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneIndexHelper.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneIndexHelper.java?rev=1588058&r1=1588057&r2=1588058&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneIndexHelper.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneIndexHelper.java Wed Apr 16 20:20:09 2014
@@ -29,6 +29,7 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXCLUDE_PROPERTY_NAMES;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXPERIMENTAL_STORAGE;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PERSISTENCE_FILE;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PERSISTENCE_NAME;
@@ -68,13 +69,22 @@ public class LuceneIndexHelper {
public static NodeBuilder newLuceneIndexDefinition(
@Nonnull NodeBuilder index, @Nonnull String name,
@Nullable Set<String> propertyTypes) {
- return newLuceneIndexDefinition(index, name, propertyTypes, null, null);
+ return newLuceneIndexDefinition(index, name, propertyTypes, null, null, null);
}
public static NodeBuilder newLuceneIndexDefinition(
@Nonnull NodeBuilder index, @Nonnull String name,
@Nullable Set<String> propertyTypes,
@Nullable Set<String> excludes, @Nullable String async) {
+ return newLuceneIndexDefinition(index, name, propertyTypes, excludes,
+ async, null);
+ }
+
+ public static NodeBuilder newLuceneIndexDefinition(
+ @Nonnull NodeBuilder index, @Nonnull String name,
+ @Nullable Set<String> propertyTypes,
+ @Nullable Set<String> excludes, @Nullable String async,
+ @Nullable Boolean stored) {
if (index.hasChildNode(name)) {
return index.child(name);
}
@@ -93,6 +103,9 @@ public class LuceneIndexHelper {
index.setProperty(createProperty(EXCLUDE_PROPERTY_NAMES, excludes,
STRINGS));
}
+ if (stored != null) {
+ index.setProperty(createProperty(EXPERIMENTAL_STORAGE, stored));
+ }
return index;
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneInitializerHelper.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneInitializerHelper.java?rev=1588058&r1=1588057&r2=1588058&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneInitializerHelper.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneInitializerHelper.java Wed Apr 16 20:20:09 2014
@@ -37,30 +37,37 @@ public class LuceneInitializerHelper imp
private String async = null;
+ private Boolean storageEnabled;
+
public LuceneInitializerHelper(String name) {
- this(name, LuceneIndexHelper.JR_PROPERTY_INCLUDES, null, null);
+ this(name, LuceneIndexHelper.JR_PROPERTY_INCLUDES, null, null, null);
+ }
+
+ public LuceneInitializerHelper(String name, Boolean storageEnabled) {
+ this(name, LuceneIndexHelper.JR_PROPERTY_INCLUDES, null, null, storageEnabled);
}
public LuceneInitializerHelper(String name, Set<String> propertyTypes) {
- this(name, propertyTypes, null, null);
+ this(name, propertyTypes, null, null, null);
}
public LuceneInitializerHelper(String name, Set<String> propertyTypes,
Set<String> excludes) {
- this(name, propertyTypes, excludes, null);
+ this(name, propertyTypes, excludes, null, null);
}
public LuceneInitializerHelper(String name, Set<String> propertyTypes,
String filePath) {
- this(name, propertyTypes, null, filePath);
+ this(name, propertyTypes, null, filePath, null);
}
public LuceneInitializerHelper(String name, Set<String> propertyTypes,
- Set<String> excludes, String filePath) {
+ Set<String> excludes, String filePath, Boolean storageEnabled) {
this.name = name;
this.propertyTypes = propertyTypes;
this.excludes = excludes;
this.filePath = filePath;
+ this.storageEnabled = storageEnabled;
}
public LuceneInitializerHelper async() {
@@ -74,9 +81,8 @@ public class LuceneInitializerHelper imp
&& builder.getChildNode(INDEX_DEFINITIONS_NAME).hasChildNode(name)) {
// do nothing
} else if (filePath == null) {
- newLuceneIndexDefinition(
- builder.child(INDEX_DEFINITIONS_NAME),
- name, propertyTypes, excludes, async);
+ newLuceneIndexDefinition(builder.child(INDEX_DEFINITIONS_NAME),
+ name, propertyTypes, excludes, async, storageEnabled);
} else {
newLuceneFileIndexDefinition(
builder.child(INDEX_DEFINITIONS_NAME),
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java?rev=1588058&r1=1588057&r2=1588058&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java Wed Apr 16 20:20:09 2014
@@ -21,6 +21,7 @@ import static org.apache.jackrabbit.JcrC
import static org.apache.jackrabbit.JcrConstants.NT_FILE;
import java.util.Properties;
+import java.util.Set;
import javax.jcr.RepositoryException;
@@ -43,7 +44,8 @@ public class LuceneOakRepositoryStub ext
@Override
protected void preCreateRepository(Jcr jcr) {
LuceneIndexProvider provider = new LowCostLuceneIndexProvider();
- jcr.with(new LuceneInitializerHelper("luceneGlobal", null))
+ jcr.with(
+ new LuceneInitializerHelper("luceneGlobal", (Set<String>) null))
.with(AggregateIndexProvider.wrap(provider.with(getNodeAggregator())))
.with((Observer) provider)
.with(new LuceneIndexEditorProvider());