You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mk...@apache.org on 2020/03/31 14:32:09 UTC
svn commit: r1875940 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/FacetCacheTest.java
Author: mkataria
Date: Tue Mar 31 14:32:08 2020
New Revision: 1875940
URL: http://svn.apache.org/viewvc?rev=1875940&view=rev
Log:
OAK-8978: Cache facet results
Added:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/FacetCacheTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1875940&r1=1875939&r2=1875940&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Tue Mar 31 14:32:08 2020
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Deque;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -196,8 +197,12 @@ public class LucenePropertyIndex extends
private static boolean NON_LAZY = Boolean.getBoolean("oak.lucene.nonLazyIndex");
public final static String OLD_FACET_PROVIDER_CONFIG_NAME = "oak.lucene.oldFacetProvider";
private final static boolean OLD_FACET_PROVIDER = Boolean.getBoolean(OLD_FACET_PROVIDER_CONFIG_NAME);
+ public final static String CACHE_FACET_RESULTS_NAME = "oak.lucene.cacheFacetResults";
+ private final boolean CACHE_FACET_RESULTS =
+ Boolean.parseBoolean(System.getProperty(CACHE_FACET_RESULTS_NAME, "true"));
private static double MIN_COST = 2.1;
+ private static boolean FLAG_CACHE_FACET_RESULTS_CHANGE = true;
private static final Logger LOG = LoggerFactory
.getLogger(LucenePropertyIndex.class);
@@ -232,6 +237,15 @@ public class LucenePropertyIndex extends
this.tracker = tracker;
this.scorerProviderFactory = factory;
this.augmentorFactory = augmentorFactory;
+ logConfigsOnce();
+ }
+
+ private void logConfigsOnce() {
+ if (FLAG_CACHE_FACET_RESULTS_CHANGE) {
+ LOG.info(OLD_FACET_PROVIDER_CONFIG_NAME + " = " + OLD_FACET_PROVIDER);
+ LOG.info(CACHE_FACET_RESULTS_NAME + " = " + CACHE_FACET_RESULTS);
+ FLAG_CACHE_FACET_RESULTS_CHANGE = false;
+ }
}
@Override
@@ -730,7 +744,7 @@ public class LucenePropertyIndex extends
protected String getType() {
return TYPE_LUCENE;
}
-
+
@Override
protected boolean filterReplacedIndexes() {
return tracker.getMountInfoProvider().hasNonDefaultMounts();
@@ -1574,11 +1588,12 @@ public class LucenePropertyIndex extends
return Iterators.concat(propIndex.iterator(), itr);
}
- static class DelayedLuceneFacetProvider implements FacetProvider {
+ class DelayedLuceneFacetProvider implements FacetProvider {
private final LucenePropertyIndex index;
private final Query query;
private final IndexPlan plan;
private final SecureFacetConfiguration config;
+ private final Map<String, List<Facet>> cachedResults = new HashMap<>();
DelayedLuceneFacetProvider(LucenePropertyIndex index, Query query, IndexPlan plan, SecureFacetConfiguration config) {
this.index = index;
@@ -1589,6 +1604,22 @@ public class LucenePropertyIndex extends
@Override
public List<Facet> getFacets(int numberOfFacets, String columnName) throws IOException {
+ if (!CACHE_FACET_RESULTS) {
+ LOG.trace(CACHE_FACET_RESULTS_NAME + " = " + CACHE_FACET_RESULTS + " getting uncached results for columnName = " + columnName);
+ return getFacetsUncached(numberOfFacets, columnName);
+ }
+ String cacheKey = columnName + "/" + numberOfFacets;
+ if (cachedResults.containsKey(cacheKey)) {
+ LOG.trace("columnName = " + columnName + " returning Facet Data from cache.");
+ return cachedResults.get(cacheKey);
+ }
+ LOG.trace("columnName = " + columnName + " facet Data not present in cache...");
+ List<Facet> result = getFacetsUncached(numberOfFacets, columnName);
+ cachedResults.put(cacheKey, result);
+ return result;
+ }
+
+ private List<Facet> getFacetsUncached(int numberOfFacets, String columnName) throws IOException {
LuceneIndexNode indexNode = index.acquireIndexNode(plan);
try {
IndexSearcher searcher = indexNode.getSearcher();
@@ -1600,7 +1631,7 @@ public class LucenePropertyIndex extends
if (topChildren != null) {
for (LabelAndValue lav : topChildren.labelValues) {
res.add(new Facet(
- lav.label, lav.value.intValue()
+ lav.label, lav.value.intValue()
));
}
return res.build();
Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/FacetCacheTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/FacetCacheTest.java?rev=1875940&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/FacetCacheTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/FacetCacheTest.java Tue Mar 31 14:32:08 2020
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid;
+
+import ch.qos.logback.classic.Level;
+import org.apache.jackrabbit.oak.InitialContent;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.api.ContentRepository;
+import org.apache.jackrabbit.oak.api.Tree;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
+import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexNodeManager;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LucenePropertyIndex;
+import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil;
+import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.OptionalEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReaderFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.query.AbstractQueryTest;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
+import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.jetbrains.annotations.Nullable;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import javax.jcr.GuestCredentials;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+import javax.jcr.query.RowIterator;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor;
+import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_FACETS;
+import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT;
+import static org.apache.jackrabbit.oak.spi.mount.Mounts.defaultMountInfoProvider;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+public class FacetCacheTest extends AbstractQueryTest {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target"));
+
+ private static final int NUM_LABELS = 4;
+ private static final int NUM_LEAF_NODES = STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT;
+ private static final String FACET_PROP = "facets";
+ private static final long REFRESH_DELTA = TimeUnit.SECONDS.toMillis(1);
+
+ private ExecutorService executorService = Executors.newFixedThreadPool(2);
+ private OptionalEditorProvider optionalEditorProvider = new OptionalEditorProvider();
+ private NRTIndexFactory nrtIndexFactory;
+ private LuceneIndexProvider luceneIndexProvider;
+ private NodeStore nodeStore;
+ private DocumentQueue queue;
+ private Clock clock = new Clock.Virtual();
+ private Whiteboard wb;
+ private QueryManager qm;
+ private Repository jcrRepo;
+ private Jcr jcr;
+ private Oak oak;
+ private Thread thread;
+ // backup original system properties i.e. before test started
+ private final Properties backupProperties = (Properties) System.getProperties().clone();
+
+ @After
+ public void tearDown() throws IOException {
+ luceneIndexProvider.close();
+ new ExecutorCloser(executorService).close();
+ nrtIndexFactory.close();
+ // restore original system properties i.e. before test started
+ System.setProperties(backupProperties);
+ }
+
+ @Override
+ protected ContentRepository createRepository() {
+ IndexCopier copier;
+ try {
+ copier = new IndexCopier(executorService, temporaryFolder.getRoot());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ MountInfoProvider mip = defaultMountInfoProvider();
+ nrtIndexFactory = new NRTIndexFactory(copier, clock, TimeUnit.MILLISECONDS.toSeconds(REFRESH_DELTA), StatisticsProvider.NOOP);
+ nrtIndexFactory.setAssertAllResourcesClosed(true);
+ LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier);
+ IndexTracker tracker = new IndexTracker(indexReaderFactory, nrtIndexFactory);
+ luceneIndexProvider = new LuceneIndexProvider(tracker);
+ queue = new DocumentQueue(100, tracker, sameThreadExecutor());
+ LuceneIndexEditorProvider editorProvider = new LuceneIndexEditorProvider(copier,
+ tracker,
+ null,
+ null,
+ mip);
+ editorProvider.setIndexingQueue(queue);
+ LocalIndexObserver localIndexObserver = new LocalIndexObserver(queue, StatisticsProvider.NOOP);
+ nodeStore = new MemoryNodeStore();
+ oak = new Oak(nodeStore)
+ .with(new InitialContent())
+ .with(new OpenSecurityProvider())
+ .with((QueryIndexProvider) luceneIndexProvider)
+ .with((Observer) luceneIndexProvider)
+ .with(localIndexObserver)
+ .with(editorProvider)
+ .with(new PropertyIndexEditorProvider())
+ .with(new NodeTypeIndexProvider())
+ .with(optionalEditorProvider)
+ .with(new NodeCounterEditorProvider())
+ //Effectively disable async indexing auto run
+ //such that we can control run timing as per test requirement
+ .withAsyncIndexing("async", TimeUnit.DAYS.toSeconds(1));
+
+ wb = oak.getWhiteboard();
+ ContentRepository repo = oak.createContentRepository();
+ return repo;
+ }
+
+ private void createSmallDataset(int k) throws RepositoryException {
+ Random rGen = new Random(42);
+ Tree par = createPath("/parent" + k);
+ par.setProperty("foo", "bar");
+ for (int i = 0; i < NUM_LABELS * 2; i++) {
+ Tree subPar = par.addChild("par" + i);
+ for (int j = 0; j < NUM_LEAF_NODES / (2 * NUM_LABELS); j++) {
+ Tree child = subPar.addChild("c" + j);
+ child.setProperty("cons", "val");
+ int labelNum = rGen.nextInt(NUM_LABELS);
+ child.setProperty("foo", "l" + labelNum);
+ }
+ }
+ }
+
+ @Test
+ public void cachedFacetTest() throws Exception {
+ LogCustomizer custom = LogCustomizer
+ .forLogger(
+ LucenePropertyIndex.class.getName())
+ .enable(Level.TRACE).create();
+ System.setProperty(LucenePropertyIndex.CACHE_FACET_RESULTS_NAME, "true");
+ System.setProperty(LuceneIndexNodeManager.OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_NAME, "40");
+ String idxName = "hybridtest";
+ Tree idx = createIndex(root.getTree("/"), idxName);
+ TestUtil.enableIndexingMode(idx, FulltextIndexConstants.IndexingMode.NRT);
+ setTraversalEnabled(false);
+ root.commit();
+ jcr = new Jcr(oak);
+ jcrRepo = jcr.createRepository();
+ createSmallDataset(0);
+ root.commit();
+ runAsyncIndex();
+ Session anonSession = jcrRepo.login(new GuestCredentials());
+ qm = anonSession.getWorkspace().getQueryManager();
+
+ try {
+ custom.starting();
+ Query q = qm.createQuery("SELECT [rep:facet(foo)] FROM [nt:base] WHERE [cons] = 'val'", SQL2);
+ QueryResult qr = q.execute();
+
+ RowIterator it;
+ try {
+ it = qr.getRows();
+ } catch (Exception e) {
+ throw e;
+ }
+ List<String> logs = custom.getLogs();
+ assertThat("Log should contain ", logs.toString(),
+ containsString("facet Data not present in cache..."));
+
+ String firstColumnName = qr.getColumnNames()[0];
+ if (it.hasNext()) {
+ Value v = it.nextRow().getValue(firstColumnName);
+ }
+ assertThat("Log should contain ", logs.toString(),
+ containsString("returning Facet Data from cache"));
+ } finally {
+ custom.finished();
+ }
+ }
+
+
+ @Test
+ public void unCachedFacetTest() throws Exception {
+ LogCustomizer custom = LogCustomizer
+ .forLogger(
+ LucenePropertyIndex.class.getName())
+ .enable(Level.TRACE).create();
+ System.setProperty(LucenePropertyIndex.CACHE_FACET_RESULTS_NAME, "false");
+ System.setProperty(LuceneIndexNodeManager.OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_NAME, "40");
+ String idxName = "hybridtest";
+ Tree idx = createIndex(root.getTree("/"), idxName);
+ TestUtil.enableIndexingMode(idx, FulltextIndexConstants.IndexingMode.NRT);
+ setTraversalEnabled(false);
+ root.commit();
+ jcr = new Jcr(oak);
+ jcrRepo = jcr.createRepository();
+ createSmallDataset(0);
+ root.commit();
+ runAsyncIndex();
+ Session anonSession = jcrRepo.login(new GuestCredentials());
+ qm = anonSession.getWorkspace().getQueryManager();
+
+ try {
+ custom.starting();
+ Query q = qm.createQuery("SELECT [rep:facet(foo)] FROM [nt:base] WHERE [cons] = 'val'", SQL2);
+ QueryResult qr = q.execute();
+
+ RowIterator it;
+ try {
+ it = qr.getRows();
+ } catch (Exception e) {
+ throw e;
+ }
+ List<String> logs = custom.getLogs();
+ String firstColumnName = qr.getColumnNames()[0];
+ if (it.hasNext()) {
+ Value v = it.nextRow().getValue(firstColumnName);
+ }
+ assertThat("Log should contain ", logs.toString(),
+ containsString(LucenePropertyIndex.CACHE_FACET_RESULTS_NAME + " = " + false + " getting uncached results for columnName = "));
+ } finally {
+ custom.finished();
+ }
+ }
+
+ private void runAsyncIndex() {
+ AsyncIndexUpdate async = (AsyncIndexUpdate) WhiteboardUtils.getService(wb, Runnable.class, new Predicate<Runnable>() {
+ @Override
+ public boolean test(@Nullable Runnable input) {
+ return input instanceof AsyncIndexUpdate;
+ }
+ });
+ assertNotNull(async);
+ async.run();
+ if (async.isFailing()) {
+ fail("AsyncIndexUpdate failed");
+ }
+ root.refresh();
+ }
+
+ private Tree createPath(String path) {
+ Tree base = root.getTree("/");
+ for (String e : PathUtils.elements(path)) {
+ base = base.addChild(e);
+ }
+ return base;
+ }
+
+ private Tree createIndex(Tree index, String name) throws RepositoryException {
+ IndexDefinitionBuilder idxBuilder = new IndexDefinitionBuilder();
+ idxBuilder.noAsync()
+ .indexRule("nt:base")
+ .property("cons").propertyIndex()
+ .property("foo").propertyIndex()
+ .getBuilderTree().setProperty(PROP_FACETS, true);
+ Tree facetConfig = idxBuilder.getBuilderTree().addChild(FACET_PROP);
+ facetConfig.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
+ facetConfig.setProperty("secure", "statistical");
+ facetConfig.setProperty("topChildren", "100");
+ Tree idxTree = index.getChild("oak:index").addChild(name);
+ idxBuilder.build(idxTree);
+ return idxTree;
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/FacetCacheTest.java
------------------------------------------------------------------------------
svn:eol-style = native