You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/07/25 07:00:29 UTC
svn commit: r1802892 - in /jackrabbit/oak/trunk:
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/
oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/
oak-run/src/main/java/org/apache/jackrabbit/oak/index/
Author: chetanm
Date: Tue Jul 25 07:00:28 2017
New Revision: 1802892
URL: http://svn.apache.org/viewvc?rev=1802892&view=rev
Log:
OAK-6471 - Support adding or updating index definitions via oak-run
Handle the import of new index definitions properly
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexDefinitionUpdater.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporter.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterProvider.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexDefinitionUpdaterTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterTest.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexDefinitionUpdater.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexDefinitionUpdater.java?rev=1802892&r1=1802891&r2=1802892&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexDefinitionUpdater.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexDefinitionUpdater.java Tue Jul 25 07:00:28 2017
@@ -22,6 +22,7 @@ package org.apache.jackrabbit.oak.plugin
import java.io.File;
import java.io.IOException;
import java.util.Map;
+import java.util.Set;
import com.google.common.base.Charsets;
import com.google.common.collect.Maps;
@@ -51,52 +52,61 @@ public class IndexDefinitionUpdater {
*/
public static final String INDEX_DEFINITIONS_JSON = "index-definitions.json";
private final Logger log = LoggerFactory.getLogger(getClass());
- private final File file;
+ private final Map<String, NodeState> indexNodeStates;
- public IndexDefinitionUpdater(File file) {
+ public IndexDefinitionUpdater(File file) throws IOException {
checkArgument(file.exists() && file.canRead(), "File [%s] cannot be read", file);
- this.file = file;
+ this.indexNodeStates = getIndexDefnStates(FileUtils.readFileToString(file, Charsets.UTF_8));
}
- public void apply(NodeState root, NodeBuilder builder) throws IOException, CommitFailedException {
- for (Map.Entry<String, NodeState> cne : getIndexDefnStates().entrySet()) {
- String indexPath = cne.getKey();
-
- if (!indexPath.startsWith("/")) {
- String msg = String.format("Invalid format of index definitions. The key name [%s] should " +
- "be index path ", indexPath);
- throw new IllegalArgumentException(msg);
- }
-
- String indexNodeName = PathUtils.getName(indexPath);
+ public IndexDefinitionUpdater(String json) throws IOException {
+ this.indexNodeStates = getIndexDefnStates(json);
+ }
- NodeState newDefinition = cne.getValue();
- String parentPath = PathUtils.getParentPath(indexPath);
- NodeState parent = NodeStateUtils.getNode(root, parentPath);
+ public void apply(NodeBuilder rootBuilder) throws IOException, CommitFailedException {
+ for (Map.Entry<String, NodeState> cne : indexNodeStates.entrySet()) {
+ String indexPath = cne.getKey();
+ apply(rootBuilder, indexPath);
+ }
+ }
- checkState(parent.exists(), "Parent node at path [%s] not found while " +
- "adding new index definition for [%s]. Intermediate paths node must exist for new index " +
- "nodes to be created", parentPath,indexPath);
+ public NodeBuilder apply(NodeBuilder rootBuilder, String indexPath) {
+ String indexNodeName = PathUtils.getName(indexPath);
- NodeState existing = parent.getChildNode(indexNodeName);
+ NodeState newDefinition = indexNodeStates.get(indexPath);
+ String parentPath = PathUtils.getParentPath(indexPath);
+ NodeState parent = NodeStateUtils.getNode(rootBuilder.getBaseState(), parentPath);
+
+ checkState(parent.exists(), "Parent node at path [%s] not found while " +
+ "adding new index definition for [%s]. Intermediate paths node must exist for new index " +
+ "nodes to be created", parentPath,indexPath);
+
+ NodeState indexDefinitionNode = parent.getChildNode(indexNodeName);
+
+ if (indexDefinitionNode.exists()) {
+ log.info("Updating index definition at path [{}]. Changes are ", indexPath);
+ String diff = JsopDiff.diffToJsop(cloneVisibleState(indexDefinitionNode), cloneVisibleState(newDefinition));
+ log.info(diff);
+ } else {
+ log.info("Adding new index definition at path [{}]", indexPath);
+ }
- if (!existing.exists()) {
- log.info("Adding new index definition at path [{}]", indexPath);
- } else {
- log.info("Updating index definition at path [{}]. Changes are ", indexPath);
- String diff = JsopDiff.diffToJsop(cloneVisibleState(existing), cloneVisibleState(newDefinition));
- log.info(diff);
- }
+ NodeBuilder indexBuilderParent = childBuilder(rootBuilder, parentPath);
+ //TODO Need to update parent :childNode list if parent has orderable children
+ indexBuilderParent.setChildNode(indexNodeName, newDefinition);
+ return indexBuilderParent.getChildNode(indexNodeName);
+ }
+ public Set<String> getIndexPaths(){
+ return indexNodeStates.keySet();
+ }
- NodeBuilder indexBuilder = childBuilder(builder, parentPath);
- indexBuilder.setChildNode(indexNodeName, newDefinition);
- }
+ public NodeState getIndexState(String indexPath){
+ return indexNodeStates.getOrDefault(indexPath, EMPTY_NODE);
}
- private Map<String, NodeState> getIndexDefnStates() throws IOException {
- String json = FileUtils.readFileToString(file, Charsets.UTF_8);
+ private static Map<String, NodeState> getIndexDefnStates(String json) throws IOException {
Base64BlobSerializer blobHandler = new Base64BlobSerializer();
Map<String, NodeState> indexDefns = Maps.newHashMap();
JsopReader reader = new JsopTokenizer(json);
@@ -104,6 +114,13 @@ public class IndexDefinitionUpdater {
if (!reader.matches('}')) {
do {
String indexPath = reader.readString();
+
+ if (!indexPath.startsWith("/")) {
+ String msg = String.format("Invalid format of index definitions. The key name [%s] should " +
+ "be index path ", indexPath);
+ throw new IllegalArgumentException(msg);
+ }
+
reader.read(':');
if (reader.matches('{')) {
JsonDeserializer deserializer = new JsonDeserializer(blobHandler);
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporter.java?rev=1802892&r1=1802891&r2=1802892&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporter.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporter.java Tue Jul 25 07:00:28 2017
@@ -40,6 +40,7 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.commit.VisibleEditor;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,7 +48,7 @@ import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_COUNT;
-import static org.apache.jackrabbit.oak.plugins.index.importer.NodeStoreUtils.childBuilder;
+import static org.apache.jackrabbit.oak.plugins.index.importer.IndexDefinitionUpdater.INDEX_DEFINITIONS_JSON;
import static org.apache.jackrabbit.oak.plugins.index.importer.NodeStoreUtils.mergeWithConcurrentCheck;
public class IndexImporter {
@@ -62,10 +63,11 @@ public class IndexImporter {
private final Map<String, IndexImporterProvider> importers = new HashMap<>();
private final IndexerInfo indexerInfo;
private final Map<String, File> indexes;
- private ListMultimap<String, IndexInfo> asyncLaneToIndexMapping;
+ private final ListMultimap<String, IndexInfo> asyncLaneToIndexMapping;
private final NodeState indexedState;
private final IndexEditorProvider indexEditorProvider;
private final AsyncIndexerLock indexerLock;
+ private final IndexDefinitionUpdater indexDefinitionUpdater;
public IndexImporter(NodeStore nodeStore, File indexDir, IndexEditorProvider indexEditorProvider,
AsyncIndexerLock indexerLock) throws IOException {
@@ -79,6 +81,8 @@ public class IndexImporter {
indexes = indexerInfo.getIndexes();
indexedState = checkNotNull(nodeStore.retrieve(indexerInfo.checkpoint), "Cannot retrieve " +
"checkpointed state [%s]", indexerInfo.checkpoint);
+ this.indexDefinitionUpdater = new IndexDefinitionUpdater(new File(indexDir, INDEX_DEFINITIONS_JSON));
+ this.asyncLaneToIndexMapping = mapIndexesToLanes(indexes);
}
public void importIndex() throws IOException, CommitFailedException {
@@ -111,32 +115,37 @@ public class IndexImporter {
NodeState root = nodeStore.getRoot();
NodeBuilder builder = root.builder();
- //Import the updated index definitions from json
- new IndexDefinitionUpdater(new File(indexDir, IndexDefinitionUpdater.INDEX_DEFINITIONS_JSON)).apply(root, builder);
-
- asyncLaneToIndexMapping = mapIndexesToLanes(indexes, builder);
-
for (IndexInfo indexInfo : asyncLaneToIndexMapping.values()){
- NodeBuilder idxBuilder = NodeStoreUtils.childBuilder(builder, indexInfo.indexPath);
- AsyncLaneSwitcher.switchLane(idxBuilder, AsyncLaneSwitcher.getTempLaneName(indexInfo.asyncLaneName));
+ if (!indexInfo.newIndex) {
+ NodeBuilder idxBuilder = NodeStoreUtils.childBuilder(builder, indexInfo.indexPath);
+ AsyncLaneSwitcher.switchLane(idxBuilder, AsyncLaneSwitcher.getTempLaneName(indexInfo.asyncLaneName));
+ }
}
mergeWithConcurrentCheck(nodeStore, builder);
}
void importIndexData() throws CommitFailedException, IOException {
NodeState root = nodeStore.getRoot();
- NodeBuilder builder = root.builder();
+ NodeBuilder rootBuilder = root.builder();
for (IndexInfo indexInfo : asyncLaneToIndexMapping.values()) {
log.info("Importing index data for {}", indexInfo.indexPath);
- NodeBuilder idxBuilder = NodeStoreUtils.childBuilder(builder, indexInfo.indexPath);
- //TODO Drop existing hidden folders. Be careful to not touch read only mount paths
+ NodeBuilder idxBuilder = indexDefinitionUpdater.apply(rootBuilder, indexInfo.indexPath);
+
+ if (indexInfo.newIndex) {
+ AsyncLaneSwitcher.switchLane(idxBuilder, AsyncLaneSwitcher.getTempLaneName(indexInfo.asyncLaneName));
+ } else {
+ //For existing ind
+ NodeState existing = NodeStateUtils.getNode(root, indexInfo.indexPath);
+ copyLaneProps(existing, idxBuilder);
+ }
+ //TODO How to support CompositeNodeStore where some of the child nodes would be hidden
incrementReIndexCount(idxBuilder);
getImporter(indexInfo.type).importIndex(root, idxBuilder, indexInfo.indexDir);
}
- mergeWithConcurrentCheck(nodeStore, builder);
+ mergeWithConcurrentCheck(nodeStore, rootBuilder);
}
- void bringIndexUpToDate() throws CommitFailedException {
+ private void bringIndexUpToDate() throws CommitFailedException {
for (String laneName : asyncLaneToIndexMapping.keySet()) {
if (ASYNC_LANE_SYNC.equals(laneName)){
continue; //TODO Handle sync indexes
@@ -218,28 +227,44 @@ public class IndexImporter {
return checkNotNull(provider, "No IndexImporterProvider found for type [%s]", type);
}
- private ListMultimap<String, IndexInfo> mapIndexesToLanes(Map<String, File> indexes, NodeBuilder rootBuilder) {
+ private ListMultimap<String, IndexInfo> mapIndexesToLanes(Map<String, File> indexes) {
+ NodeState rootState = nodeStore.getRoot();
ListMultimap<String, IndexInfo> map = ArrayListMultimap.create();
for (Map.Entry<String, File> e : indexes.entrySet()) {
String indexPath = e.getKey();
- NodeBuilder indexBuilder = childBuilder(rootBuilder, indexPath);
- checkArgument(indexBuilder.exists(), "No index node found at path [%s]", indexPath);
+ NodeState indexState = indexDefinitionUpdater.getIndexState(indexPath);
+ checkArgument(indexState.exists(), "No index node found at path [%s]", indexPath);
- String type = indexBuilder.getString(IndexConstants.TYPE_PROPERTY_NAME);
+ boolean newIndex = !NodeStateUtils.getNode(rootState, indexPath).exists();
+
+ String type = indexState.getString(IndexConstants.TYPE_PROPERTY_NAME);
checkNotNull(type, "No 'type' property found for index at path [%s]", indexPath);
- String asyncName = getAsyncLaneName(indexPath, indexBuilder.getNodeState());
+ String asyncName = getAsyncLaneName(indexPath, indexState);
if (asyncName == null) {
asyncName = ASYNC_LANE_SYNC;
}
- map.put(asyncName, new IndexInfo(indexPath, e.getValue(), asyncName, type));
+ map.put(asyncName, new IndexInfo(indexPath, e.getValue(), asyncName, type, newIndex));
}
return map;
}
+ private static void copyLaneProps(NodeState existing, NodeBuilder indexBuilder) {
+ copy(IndexConstants.ASYNC_PROPERTY_NAME, existing, indexBuilder);
+ copy(AsyncLaneSwitcher.ASYNC_PREVIOUS, existing, indexBuilder);
+ }
+
+ private static void copy(String propName, NodeState existing, NodeBuilder indexBuilder) {
+ PropertyState ps = existing.getProperty(propName);
+ if (ps != null) {
+ indexBuilder.setProperty(ps);
+ }
+ }
+
+
/**
* Determines the async lane name. This method also check if lane was previously switched
* then it uses the actual lane name prior to switch was done
@@ -279,12 +304,14 @@ public class IndexImporter {
final File indexDir;
final String asyncLaneName;
final String type;
+ final boolean newIndex;
- private IndexInfo(String indexPath, File indexDir, String asyncLaneName, String type) {
+ private IndexInfo(String indexPath, File indexDir, String asyncLaneName, String type, boolean newIndex) {
this.indexPath = indexPath;
this.indexDir = indexDir;
this.asyncLaneName = asyncLaneName;
this.type = type;
+ this.newIndex = newIndex;
}
@Override
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterProvider.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterProvider.java?rev=1802892&r1=1802891&r2=1802892&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterProvider.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterProvider.java Tue Jul 25 07:00:28 2017
@@ -22,6 +22,7 @@ package org.apache.jackrabbit.oak.plugin
import java.io.File;
import java.io.IOException;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
@@ -35,7 +36,7 @@ public interface IndexImporterProvider {
* Import the index data from given directory into the
* NodeBuilder created for the index at given path
*/
- void importIndex(NodeState root, NodeBuilder defn, File indexDir) throws IOException;
+ void importIndex(NodeState root, NodeBuilder defn, File indexDir) throws IOException, CommitFailedException;
/**
* Index type for this implementation
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexDefinitionUpdaterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexDefinitionUpdaterTest.java?rev=1802892&r1=1802891&r2=1802892&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexDefinitionUpdaterTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexDefinitionUpdaterTest.java Tue Jul 25 07:00:28 2017
@@ -95,17 +95,35 @@ public class IndexDefinitionUpdaterTest
map.put("a", ImmutableMap.of("a2", "b2"));
String json = JSONObject.toJSONString(map);
applyJson(json);
-
}
- private void applyJson(String json) throws IOException, CommitFailedException {
- NodeBuilder builder;File file = folder.newFile();
- Files.write(json, file, UTF_8);
+ @Test
+ public void applyToIndexPath() throws Exception{
+ String json = "{\"/oak:index/barIndex\": {\n" +
+ " \"compatVersion\": 2,\n" +
+ " \"type\": \"lucene\",\n" +
+ " \"barIndexProp\": \"barbar\",\n" +
+ " \"async\": \"async\",\n" +
+ " \"jcr:primaryType\": \"oak:QueryIndexDefinition\"\n" +
+ " }}";
- IndexDefinitionUpdater update = new IndexDefinitionUpdater(file);
+ NodeBuilder builder = store.getRoot().builder();
+ builder.child("oak:index");
+ store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ IndexDefinitionUpdater updater = new IndexDefinitionUpdater(json);
builder = store.getRoot().builder();
- update.apply(store.getRoot(), builder);
+ NodeBuilder idxBuilder = updater.apply(builder, "/oak:index/barIndex");
+
+ //Check builder returned is of /oak:index/barIndex
+ assertTrue(idxBuilder.hasProperty("barIndexProp"));
+ }
+
+ private void applyJson(String json) throws IOException, CommitFailedException {
+ IndexDefinitionUpdater update = new IndexDefinitionUpdater(json);
+
+ NodeBuilder builder = store.getRoot().builder();
+ update.apply(builder);
store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterTest.java?rev=1802892&r1=1802891&r2=1802892&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterTest.java Tue Jul 25 07:00:28 2017
@@ -39,6 +39,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback;
+import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider;
import org.apache.jackrabbit.oak.plugins.index.inventory.IndexDefinitionPrinter;
import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexLookup;
@@ -51,6 +52,7 @@ import org.apache.jackrabbit.oak.query.a
import org.apache.jackrabbit.oak.query.index.FilterImpl;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.Editor;
+import org.apache.jackrabbit.oak.spi.commit.EditorHook;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.query.Filter;
import org.apache.jackrabbit.oak.spi.query.QueryEngineSettings;
@@ -66,11 +68,13 @@ import static com.google.common.base.Cha
import static com.google.common.collect.ImmutableSet.of;
import static java.util.Arrays.asList;
import static org.apache.jackrabbit.JcrConstants.NT_BASE;
+import static org.apache.jackrabbit.oak.InitialContent.INITIAL_CONTENT;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.ASYNC_PROPERTY_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_COUNT;
import static org.apache.jackrabbit.oak.plugins.index.IndexUtils.createIndexDefinition;
import static org.apache.jackrabbit.oak.plugins.index.importer.AsyncIndexerLock.NOOP_LOCK;
+import static org.apache.jackrabbit.oak.plugins.index.importer.IndexDefinitionUpdater.INDEX_DEFINITIONS_JSON;
import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
import static org.junit.Assert.*;
import static org.mockito.Mockito.mock;
@@ -250,6 +254,9 @@ public class IndexImporterTest {
importer.importIndex();
+ NodeState idx = store.getRoot().getChildNode("oak:index").getChildNode("fooIndex");
+ assertEquals("async", idx.getString("async"));
+
lookup = new PropertyIndexLookup(store.getRoot());
//It would not pickup /e as thats not yet indexed as part of last checkpoint
assertEquals(of("a", "b", "c", "d"), find(lookup, "foo", "abc", f));
@@ -257,6 +264,88 @@ public class IndexImporterTest {
}
@Test
+ public void importIndex_newIndex() throws Exception{
+ NodeBuilder builder = store.getRoot().builder();
+ builder.child("oak:index");
+ builder.child("a").setProperty("foo", "abc");
+ store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ String json = "{\"/oak:index/fooIndex\": {\n" +
+ " \"reindexCount\": 1,\n" +
+ " \"reindex\": false,\n" +
+ " \"type\": \"property\",\n" +
+ " \"async\" : \"async\",\n" +
+ " \"propertyNames\": [\"foo\"],\n" +
+ " \"jcr:primaryType\": \"oak:QueryIndexDefinition\"\n" +
+ " }\n" +
+ "}";
+
+
+ File indexFolder = temporaryFolder.getRoot();
+
+ //Create checkpoint file
+ String checkpoint = store.checkpoint(1000000);
+ IndexerInfo info = new IndexerInfo(indexFolder, checkpoint);
+ info.save();
+
+ //Create index definitions json
+ Files.write(json, new File(indexFolder, INDEX_DEFINITIONS_JSON), UTF_8);
+
+ createIndexFolder(indexFolder, "/oak:index/fooIndex");
+
+ //Prepare importer
+ IndexImporterProvider importerProvider = new IndexImporterProvider() {
+ @Override
+ public void importIndex(NodeState root, NodeBuilder defn, File indexDir)
+ throws CommitFailedException {
+ NodeState fooIndex = getFooIndexNodeState();
+ defn.setChildNode(IndexConstants.INDEX_CONTENT_NODE_NAME,
+ fooIndex.getChildNode(":index"));
+ }
+
+ @Override
+ public String getType() {
+ return "property";
+ }
+ };
+
+ IndexImporter importer = new IndexImporter(store, indexFolder, provider, NOOP_LOCK);
+ importer.addImporterProvider(importerProvider);
+
+ //Add some more indexable data
+ builder = store.getRoot().builder();
+ builder.child("c").setProperty("foo", "abc");
+ builder.child("d").setProperty("foo", "abc");
+ store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ new AsyncIndexUpdate("async", store, provider).run();
+
+ //Now perform import
+ importer.importIndex();
+
+ FilterImpl f = createFilter(store.getRoot(), NT_BASE);
+ PropertyIndexLookup lookup = new PropertyIndexLookup(store.getRoot());
+ assertEquals(of("a", "c", "d"), find(lookup, "foo", "abc", f));
+
+ NodeState idx = store.getRoot().getChildNode("oak:index").getChildNode("fooIndex");
+ assertEquals(2, idx.getLong("reindexCount"));
+ }
+
+ private NodeState getFooIndexNodeState() throws CommitFailedException {
+ NodeState root = INITIAL_CONTENT;
+ // Add index definition
+ NodeBuilder builder = root.builder();
+ NodeBuilder index = createIndexDefinition(builder.child(INDEX_DEFINITIONS_NAME), "fooIndex",
+ true, false, ImmutableSet.of("foo"), null);
+ builder.child("a").setProperty("foo", "abc");
+ NodeState after = builder.getNodeState();
+ EditorHook hook = new EditorHook(
+ new IndexUpdateProvider(new PropertyIndexEditorProvider()));
+ NodeState indexed = hook.processCommit(EMPTY_NODE, after, CommitInfo.EMPTY);
+ return indexed.getChildNode("oak:index").getChildNode("fooIndex");
+ }
+
+ @Test
public void laneName() throws Exception{
NodeBuilder builder = EMPTY_NODE.builder();
builder.setProperty(IndexConstants.ASYNC_PROPERTY_NAME, "async");
@@ -374,22 +463,26 @@ public class IndexImporterTest {
info.save();
for (String indexPath : indexPaths){
- String dirName = PathUtils.getName(indexPath);
- File indexDir = new File(temporaryFolder.getRoot(), dirName);
- File indexMeta = new File(indexDir, IndexerInfo.INDEX_METADATA_FILE_NAME);
- Properties p = new Properties();
- p.setProperty(IndexerInfo.PROP_INDEX_PATH, indexPath);
- indexDir.mkdir();
- PropUtils.writeTo(p, indexMeta, "index info");
+ createIndexFolder(temporaryFolder.getRoot(), indexPath);
}
dumpIndexDefinitions(indexPaths);
return checkpoint;
}
+ private void createIndexFolder(File indexFolder, String indexPath) throws IOException {
+ String dirName = PathUtils.getName(indexPath);
+ File indexDir = new File(indexFolder, dirName);
+ File indexMeta = new File(indexDir, IndexerInfo.INDEX_METADATA_FILE_NAME);
+ Properties p = new Properties();
+ p.setProperty(IndexerInfo.PROP_INDEX_PATH, indexPath);
+ indexDir.mkdir();
+ PropUtils.writeTo(p, indexMeta, "index info");
+ }
+
private void dumpIndexDefinitions(String... indexPaths) throws IOException, CommitFailedException {
IndexDefinitionPrinter printer = new IndexDefinitionPrinter(store, () -> asList(indexPaths));
printer.setFilter("{\"properties\":[\"*\", \"-:childOrder\"],\"nodes\":[\"*\", \"-:index-definition\"]}");
- File file = new File(temporaryFolder.getRoot(), IndexDefinitionUpdater.INDEX_DEFINITIONS_JSON);
+ File file = new File(temporaryFolder.getRoot(), INDEX_DEFINITIONS_JSON);
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
printer.print(pw, Format.JSON, false);
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java?rev=1802892&r1=1802891&r2=1802892&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java Tue Jul 25 07:00:28 2017
@@ -92,9 +92,9 @@ public class IndexerSupport {
return checkpointedState;
}
- public void updateIndexDefinitions(NodeState root, NodeBuilder builder) throws IOException, CommitFailedException {
+ public void updateIndexDefinitions(NodeBuilder rootBuilder) throws IOException, CommitFailedException {
if (indexDefinitions != null) {
- new IndexDefinitionUpdater(indexDefinitions).apply(root, builder);
+ new IndexDefinitionUpdater(indexDefinitions).apply(rootBuilder);
}
}
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java?rev=1802892&r1=1802891&r2=1802892&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java Tue Jul 25 07:00:28 2017
@@ -166,7 +166,7 @@ public class OutOfBandIndexer implements
private void switchIndexLanesAndReindexFlag() throws CommitFailedException, IOException {
NodeState root = copyOnWriteStore.getRoot();
NodeBuilder builder = root.builder();
- indexerSupport.updateIndexDefinitions(root, builder);
+ indexerSupport.updateIndexDefinitions(builder);
for (String indexPath : indexHelper.getIndexPaths()) {
//TODO Do it only for lucene indexes for now