You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/05/09 11:42:57 UTC
svn commit: r1794544 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/
Author: chetanm
Date: Tue May 9 11:42:57 2017
New Revision: 1794544
URL: http://svn.apache.org/viewvc?rev=1794544&view=rev
Log:
OAK-6192 - Lucene IndexInfoProvider implementation
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java (with props)
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProviderTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java?rev=1794544&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java Tue May 9 11:42:57 2017
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfo;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.IndexInfo;
+import org.apache.jackrabbit.oak.plugins.index.IndexInfoProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryUtils;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.MultiplexersLucene;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.spi.state.ReadOnlyBuilder;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.store.Directory;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+public class LuceneIndexInfoProvider implements IndexInfoProvider {
+
+ private final NodeStore nodeStore;
+
+ private final AsyncIndexInfoService asyncInfoService;
+
+ private final File workDir;
+
+ public LuceneIndexInfoProvider(NodeStore nodeStore, AsyncIndexInfoService asyncInfoService, File workDir) {
+ this.nodeStore = checkNotNull(nodeStore);
+ this.asyncInfoService = checkNotNull(asyncInfoService);
+ this.workDir = checkNotNull(workDir);
+ }
+
+ static String getAsyncName(NodeState idxState, String indexPath) {
+ PropertyState async = idxState.getProperty(IndexConstants.ASYNC_PROPERTY_NAME);
+ if (async != null) {
+ Set<String> asyncNames = Sets.newHashSet(async.getValue(Type.STRINGS));
+ asyncNames.remove(IndexConstants.INDEXING_MODE_NRT);
+ asyncNames.remove(IndexConstants.INDEXING_MODE_SYNC);
+ checkArgument(!asyncNames.isEmpty(), "No valid async name found for " +
+ "index [%s], definition %s", indexPath, idxState);
+ return Iterables.getOnlyElement(asyncNames);
+ }
+ return null;
+ }
+
+ @Override
+ public String getType() {
+ return LuceneIndexConstants.TYPE_LUCENE;
+ }
+
+ @Override
+ public IndexInfo getInfo(String indexPath) throws IOException {
+ NodeState idxState = NodeStateUtils.getNode(nodeStore.getRoot(), indexPath);
+
+ checkArgument(LuceneIndexConstants.TYPE_LUCENE.equals(idxState.getString(IndexConstants.TYPE_PROPERTY_NAME)),
+ "Index definition at [%s] is not of type 'lucene'", indexPath);
+
+ LuceneIndexInfo info = new LuceneIndexInfo(indexPath);
+ computeSize(idxState, info);
+ computeAsyncIndexInfo(idxState, indexPath, info);
+ return info;
+ }
+
+ @Override
+ public boolean isValid(String indexPath) throws IOException {
+ IndexConsistencyChecker checker = new IndexConsistencyChecker(nodeStore.getRoot(), indexPath, workDir);
+ return checker.check(IndexConsistencyChecker.Level.BLOBS_ONLY).clean;
+ }
+
+ private void computeAsyncIndexInfo(NodeState idxState, String indexPath, LuceneIndexInfo info) {
+ String asyncName = getAsyncName(idxState, indexPath);
+ checkNotNull(asyncName, "No 'async' value for index definition " +
+ "at [%s]. Definition %s", indexPath, idxState);
+
+ AsyncIndexInfo asyncInfo = asyncInfoService.getInfo(asyncName);
+ checkNotNull(asyncInfo, "No async info found for name [%s] " +
+ "for index at [%s]", asyncName, indexPath);
+
+ info.indexedUptoTime = asyncInfo.getLastIndexedTo();
+ info.asyncName = asyncName;
+ }
+
+ private void computeSize(NodeState idxState, LuceneIndexInfo info) throws IOException {
+ IndexDefinition defn = IndexDefinition.newBuilder(nodeStore.getRoot(), idxState, info.indexPath).build();
+ for (String dirName : idxState.getChildNodeNames()) {
+ if (NodeStateUtils.isHidden(dirName) && MultiplexersLucene.isIndexDirName(dirName)) {
+ Directory dir = new OakDirectory(new ReadOnlyBuilder(idxState), dirName, defn, true);
+ try (DirectoryReader dirReader = DirectoryReader.open(dir)) {
+ info.numEntries += dirReader.numDocs();
+ info.size = DirectoryUtils.dirSize(dir);
+ }
+ }
+ }
+ }
+
+ private static class LuceneIndexInfo implements IndexInfo {
+ String indexPath;
+ String asyncName;
+ long numEntries;
+ long size;
+ long indexedUptoTime;
+
+ public LuceneIndexInfo(String indexPath) {
+ this.indexPath = indexPath;
+ }
+
+ @Override
+ public String getIndexPath() {
+ return indexPath;
+ }
+
+ @Override
+ public String getType() {
+ return LuceneIndexConstants.TYPE_LUCENE;
+ }
+
+ @Override
+ public String getAsyncLaneName() {
+ return asyncName;
+ }
+
+ @Override
+ public long getLastUpdatedTime() {
+ return 0; //TODO To be computed
+ }
+
+ @Override
+ public long getIndexedUpToTime() {
+ return indexedUptoTime;
+ }
+
+ @Override
+ public long getEstimatedEntryCount() {
+ return numEntries;
+ }
+
+ @Override
+ public long getSizeInBytes() {
+ return size;
+ }
+
+ @Override
+ public boolean hasIndexDefinitionChangedWithoutReindexing() {
+ return false; //TODO To be computed
+ }
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProvider.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java?rev=1794544&r1=1794543&r2=1794544&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java Tue May 9 11:42:57 2017
@@ -51,7 +51,9 @@ import org.apache.jackrabbit.oak.cache.C
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard;
import org.apache.jackrabbit.oak.plugins.document.spi.JournalPropertyService;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService;
import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.IndexInfoProvider;
import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator;
import org.apache.jackrabbit.oak.plugins.index.fulltext.PreExtractedTextProvider;
@@ -260,6 +262,9 @@ public class LuceneIndexProviderService
@Reference
private IndexPathService indexPathService;
+ @Reference
+ private AsyncIndexInfoService asyncIndexInfoService;
+
@Reference(cardinality = ReferenceCardinality.OPTIONAL_UNARY,
policyOption = ReferencePolicyOption.GREEDY,
policy = ReferencePolicy.DYNAMIC
@@ -311,15 +316,20 @@ public class LuceneIndexProviderService
registerObserver(bundleContext, config);
registerLocalIndexObserver(bundleContext, tracker, config);
registerIndexEditor(bundleContext, tracker, config);
+ registerIndexInfoProvider(bundleContext);
oakRegs.add(registerMBean(whiteboard,
LuceneIndexMBean.class,
- new LuceneIndexMBeanImpl(indexProvider.getTracker(), nodeStore, indexPathService, new File(indexDir, "indexCheckDir")),
+ new LuceneIndexMBeanImpl(indexProvider.getTracker(), nodeStore, indexPathService, getIndexCheckDir()),
LuceneIndexMBean.TYPE,
"Lucene Index statistics"));
registerGCMonitor(whiteboard, indexProvider.getTracker());
}
+ private File getIndexCheckDir() {
+ return new File(checkNotNull(indexDir), "indexCheckDir");
+ }
+
@Deactivate
private void deactivate() throws InterruptedException, IOException {
for (ServiceRegistration reg : regs) {
@@ -661,6 +671,11 @@ public class LuceneIndexProviderService
}
}
+ private void registerIndexInfoProvider(BundleContext bundleContext) {
+ IndexInfoProvider infoProvider = new LuceneIndexInfoProvider(nodeStore, asyncIndexInfoService, getIndexCheckDir());
+ regs.add(bundleContext.registerService(IndexInfoProvider.class.getName(), infoProvider, null));
+ }
+
protected void bindNodeAggregator(NodeAggregator aggregator) {
this.nodeAggregator = aggregator;
initialize();
Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProviderTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProviderTest.java?rev=1794544&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProviderTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProviderTest.java Tue May 9 11:42:57 2017
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import java.io.File;
+
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfo;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService;
+import org.apache.jackrabbit.oak.plugins.index.IndexInfo;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class LuceneIndexInfoProviderTest {
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target"));
+ private NodeStore store = new MemoryNodeStore();
+ private AsyncIndexInfoService asyncService = mock(AsyncIndexInfoService.class);
+ private LuceneIndexInfoProvider provider;
+
+ @Before
+ public void setUp() {
+ provider = new LuceneIndexInfoProvider(store, asyncService, temporaryFolder.getRoot());
+ }
+
+ @Test
+ public void asyncName() throws Exception {
+ assertNull(LuceneIndexInfoProvider.getAsyncName(EMPTY_NODE, "/fooIndex"));
+
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty("async", newArrayList("async2", "sync"), Type.STRINGS);
+ assertEquals("async2", LuceneIndexInfoProvider.getAsyncName(builder.getNodeState(), "/fooIndex"));
+
+ builder.setProperty("async", newArrayList("async3"), Type.STRINGS);
+ assertEquals("async3", LuceneIndexInfoProvider.getAsyncName(builder.getNodeState(), "/fooIndex"));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void infoNonExisting() throws Exception {
+ provider.getInfo("/no/existing/path");
+ }
+
+ @Test
+ public void info() throws Exception {
+ IndexDefinitionBuilder defnBuilder = new IndexDefinitionBuilder();
+
+ NodeBuilder builder = store.getRoot().builder();
+ builder.child("oak:index").setChildNode("fooIndex", defnBuilder.build());
+ store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ when(asyncService.getInfo("async"))
+ .thenReturn(new AsyncIndexInfo("async", 0,0, false, null));
+
+ IndexInfo info = provider.getInfo("/oak:index/fooIndex");
+
+ assertNotNull(info);
+ }
+
+}
\ No newline at end of file
Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexInfoProviderTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java?rev=1794544&r1=1794543&r2=1794544&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java Tue May 9 11:42:57 2017
@@ -39,6 +39,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils;
import org.apache.jackrabbit.oak.plugins.document.spi.JournalPropertyService;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService;
import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
import org.apache.jackrabbit.oak.plugins.index.fulltext.ExtractedText;
@@ -86,6 +87,7 @@ public class LuceneIndexProviderServiceT
context.registerService(IndexAugmentorFactory.class, mock(IndexAugmentorFactory.class));
context.registerService(NodeStore.class, new MemoryNodeStore());
context.registerService(IndexPathService.class, mock(IndexPathService.class));
+ context.registerService(AsyncIndexInfoService.class, mock(AsyncIndexInfoService.class));
MockOsgi.injectServices(service, context.bundleContext());
}