You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/01/11 05:20:31 UTC
svn commit: r1778238 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/
Author: chetanm
Date: Wed Jan 11 05:20:31 2017
New Revision: 1778238
URL: http://svn.apache.org/viewvc?rev=1778238&view=rev
Log:
OAK-4808 - Index external changes as part of NRT indexing
Add implementation of JournalPropertyService for Lucene index data. as part of JournalEntry a JSON instance of
indexed doc Path -> [ set of index paths in which it is indexed]
Would be stored. This would then be accumulated in builder
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPathInfo.java (with props)
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java (with props)
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyBuilder.java (with props)
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyService.java (with props)
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyBuilderTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPathInfo.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPathInfo.java?rev=1778238&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPathInfo.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPathInfo.java Wed Jan 11 05:20:31 2017
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid;
+
+interface IndexedPathInfo {
+
+ String getPath();
+
+ Iterable<String> getIndexPaths();
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPathInfo.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java?rev=1778238&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java Wed Jan 11 05:20:31 2017
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Multimap;
+import org.apache.jackrabbit.oak.plugins.document.spi.JournalProperty;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+class IndexedPaths implements JournalProperty, Iterable<IndexedPathInfo> {
+ private final Multimap<String, String> indexedPaths;
+
+ public IndexedPaths(Multimap<String, String> indexedPaths) {
+ this.indexedPaths = checkNotNull(indexedPaths);
+ }
+
+ @Override
+ public Iterator<IndexedPathInfo> iterator() {
+ return Iterators.transform(indexedPaths.asMap().entrySet().iterator(),
+ new Function<Map.Entry<String, Collection<String>>, IndexedPathInfo>() {
+ @Override
+ public IndexedPathInfo apply(final Map.Entry<String, Collection<String>> input) {
+ return new IndexedPathInfo() {
+ @Override
+ public String getPath() {
+ return input.getKey();
+ }
+
+ @Override
+ public Iterable<String> getIndexPaths() {
+ return input.getValue();
+ }
+ };
+ }
+ });
+ }
+
+ @Override
+ public String toString() {
+ return indexedPaths.toString();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ IndexedPaths that = (IndexedPaths) o;
+
+ return indexedPaths.equals(that.indexedPaths);
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java?rev=1778238&r1=1778237&r2=1778238&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java Wed Jan 11 05:20:31 2017
@@ -22,18 +22,21 @@ package org.apache.jackrabbit.oak.plugin
import java.util.Collection;
import java.util.Map;
+import javax.annotation.Nonnull;
+
import com.google.common.base.Function;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.ListMultimap;
+import org.apache.jackrabbit.oak.plugins.document.spi.JournalProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkNotNull;
-public class LuceneDocumentHolder {
+public class LuceneDocumentHolder implements JournalProperty{
private static final Logger log = LoggerFactory.getLogger(LuceneDocumentHolder.class);
- public static final String NAME = "oak.lucene.documentHolder";
+ public static final String NAME = "luceneDocs";
private final ListMultimap<String, LuceneDoc> nrtIndexedList = ArrayListMultimap.create();
private final ListMultimap<String, LuceneDoc> syncIndexedList = ArrayListMultimap.create();
@@ -45,7 +48,7 @@ public class LuceneDocumentHolder {
private boolean docAddedToQueue;
private boolean schedulingDone;
- public LuceneDocumentHolder(IndexingQueue documentQueue, int inMemoryDocsLimit) {
+ public LuceneDocumentHolder(@Nonnull IndexingQueue documentQueue, int inMemoryDocsLimit) {
this.documentQueue = checkNotNull(documentQueue);
this.inMemoryDocsLimit = inMemoryDocsLimit;
}
@@ -64,6 +67,7 @@ public class LuceneDocumentHolder {
}
public void add(boolean sync, LuceneDoc doc) {
+ doc = checkNotNull(doc);
//First try adding to queue in non blocking manner
if (documentQueue.addIfNotFullWithoutWait(doc)){
if (sync){
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyBuilder.java?rev=1778238&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyBuilder.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyBuilder.java Wed Jan 11 05:20:31 2017
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid;
+
+import java.util.Collection;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
+import org.apache.jackrabbit.oak.commons.json.JsopReader;
+import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
+import org.apache.jackrabbit.oak.commons.json.JsopWriter;
+import org.apache.jackrabbit.oak.plugins.document.spi.JournalProperty;
+import org.apache.jackrabbit.oak.plugins.document.spi.JournalPropertyBuilder;
+
+class LuceneJournalPropertyBuilder implements JournalPropertyBuilder<LuceneDocumentHolder>{
+ //Use HashMultimap to ensure that indexPath is not duplicated per node path
+ private final Multimap<String, String> indexedNodes = HashMultimap.create();
+
+ //~---------------------------------< serialize >
+ @Override
+ public void addProperty(@Nullable LuceneDocumentHolder docHolder) {
+ if (docHolder != null){
+ for (LuceneDocInfo d : docHolder.getAllLuceneDocInfo()){
+ indexedNodes.put(d.getDocPath(), d.getIndexPath());
+ }
+ }
+ }
+
+ @Override
+ public String buildAsString() {
+ JsopWriter json = new JsopBuilder();
+ json.object();
+ for (Map.Entry<String, Collection<String>> e : indexedNodes.asMap().entrySet()){
+ json.key(e.getKey()).array();
+ for (String v : e.getValue()){
+ json.value(v);
+ }
+ json.endArray();
+ }
+ json.endObject();
+ return json.toString();
+ }
+
+ //~---------------------------------< deserialize >
+
+ @Override
+ public void addSerializedProperty(@Nullable String json) {
+ if (json == null || json.isEmpty()){
+ return;
+ }
+ //TODO Add support for overflow
+ JsopReader reader = new JsopTokenizer(json);
+ reader.read('{');
+ if (!reader.matches('}')) {
+ do {
+ String path = reader.readString();
+ reader.read(':');
+ reader.read('[');
+ for (boolean first = true; !reader.matches(']'); first = false) {
+ if (!first) {
+ reader.read(',');
+ }
+ indexedNodes.put(path, reader.readString());
+ }
+ } while (reader.matches(','));
+ reader.read('}');
+ }
+ reader.read(JsopReader.END);
+ }
+
+ @Override
+ public JournalProperty build() {
+ return new IndexedPaths(indexedNodes);
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyBuilder.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyService.java?rev=1778238&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyService.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyService.java Wed Jan 11 05:20:31 2017
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid;
+
+import org.apache.jackrabbit.oak.plugins.document.spi.JournalPropertyBuilder;
+import org.apache.jackrabbit.oak.plugins.document.spi.JournalPropertyService;
+
+public class LuceneJournalPropertyService implements JournalPropertyService {
+ @Override
+ public JournalPropertyBuilder newBuilder() {
+ return new LuceneJournalPropertyBuilder();
+ }
+
+ @Override
+ public String getName() {
+ return LuceneDocumentHolder.NAME;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyService.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyBuilderTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyBuilderTest.java?rev=1778238&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyBuilderTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyBuilderTest.java Wed Jan 11 05:20:31 2017
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+import org.junit.Test;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class LuceneJournalPropertyBuilderTest {
+
+ private LuceneJournalPropertyBuilder builder = new LuceneJournalPropertyBuilder();
+
+ @Test
+ public void nullProperty() throws Exception{
+ builder.addProperty(null);
+ assertEquals("{}", builder.buildAsString());
+ assertTrue(Iterables.isEmpty(((IndexedPaths)builder.build())));
+ }
+
+ @Test
+ public void nullOrEmptyJson() throws Exception{
+ builder.addProperty(null);
+
+ LuceneJournalPropertyBuilder builder2 = new LuceneJournalPropertyBuilder();
+ builder2.addSerializedProperty(null);
+ builder2.addSerializedProperty(builder.buildAsString());
+
+ assertTrue(Iterables.isEmpty(((IndexedPaths)builder2.build())));
+ }
+
+ @Test
+ public void addMulti() throws Exception{
+ LuceneDocumentHolder h1 = createHolder();
+ h1.add(true, LuceneDoc.forDelete("/oak:index/foo", "/a"));
+ h1.add(true, LuceneDoc.forDelete("/oak:index/foo", "/b"));
+ builder.addProperty(h1);
+
+ LuceneDocumentHolder h2 = createHolder();
+ h2.add(true, LuceneDoc.forDelete("/oak:index/bar", "/a"));
+ builder.addProperty(h2);
+
+ IndexedPaths indexedPaths = (IndexedPaths) builder.build();
+ Multimap<String, String> map = createdIndexPathMap(indexedPaths);
+ assertThat(map.keySet(), containsInAnyOrder("/oak:index/foo", "/oak:index/bar"));
+ assertThat(map.get("/oak:index/foo"), containsInAnyOrder("/a", "/b"));
+ }
+
+ @Test
+ public void addMultiJson() throws Exception{
+ LuceneDocumentHolder h1 = createHolder();
+ h1.add(true, LuceneDoc.forDelete("/oak:index/foo", "/a"));
+ h1.add(true, LuceneDoc.forDelete("/oak:index/foo", "/b"));
+ builder.addProperty(h1);
+
+ LuceneDocumentHolder h2 = createHolder();
+ h2.add(true, LuceneDoc.forDelete("/oak:index/bar", "/a"));
+ builder.addProperty(h2);
+
+ String json = builder.buildAsString();
+ LuceneJournalPropertyBuilder builder2 = new LuceneJournalPropertyBuilder();
+ builder2.addSerializedProperty(json);
+
+ IndexedPaths indexedPaths = (IndexedPaths) builder2.build();
+ Multimap<String, String> map = createdIndexPathMap(indexedPaths);
+ assertThat(map.keySet(), containsInAnyOrder("/oak:index/foo", "/oak:index/bar"));
+ assertThat(map.get("/oak:index/foo"), containsInAnyOrder("/a", "/b"));
+ }
+
+ private Multimap<String, String> createdIndexPathMap(Iterable<IndexedPathInfo> itr){
+ Multimap<String, String> map = HashMultimap.create();
+ for (IndexedPathInfo i : itr){
+ for (String indexPath : i.getIndexPaths()){
+ map.put(indexPath, i.getPath());
+ }
+ }
+ return map;
+ }
+
+ private LuceneDocumentHolder createHolder(){
+ IndexingQueue queue = mock(IndexingQueue.class);
+ when(queue.addIfNotFullWithoutWait(any(LuceneDoc.class))).thenReturn(true);
+ return new LuceneDocumentHolder(queue, 100);
+ }
+}
\ No newline at end of file
Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneJournalPropertyBuilderTest.java
------------------------------------------------------------------------------
svn:eol-style = native