You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2017/08/24 09:15:10 UTC
svn commit: r1806013 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/ te...
Author: tommaso
Date: Thu Aug 24 09:15:10 2017
New Revision: 1806013
URL: http://svn.apache.org/viewvc?rev=1806013&view=rev
Log:
OAK-5192, OAK-6514 - allow configuring mp, default to 'migitate' mp
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java (with props)
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1806013&r1=1806012&r2=1806013&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java Thu Aug 24 09:15:10 2017
@@ -52,6 +52,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.lucene.util.ConfigUtil;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.FunctionIndexProcessor;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.TokenizerChain;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.CommitMitigatingTieredMergePolicy;
import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
import org.apache.jackrabbit.oak.plugins.nodetype.ReadOnlyNodeTypeManager;
import org.apache.jackrabbit.oak.plugins.tree.TreeFactory;
@@ -67,6 +68,11 @@ import org.apache.lucene.analysis.miscel
import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.path.PathHierarchyTokenizerFactory;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.LogByteSizeMergePolicy;
+import org.apache.lucene.index.LogDocMergePolicy;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -424,6 +430,12 @@ public final class IndexDefinition imple
return codec;
}
+ @Nonnull
+ public MergePolicy getMergePolicy() {
+ // MP is not cached to avoid complaining about multiple IWs with multiplexing writers
+ return createMergePolicy();
+ }
+
public long getReindexCount(){
return reindexCount;
}
@@ -1494,6 +1506,28 @@ public final class IndexDefinition imple
return codec;
}
+ private MergePolicy createMergePolicy() {
+ String mergePolicyName = getOptionalValue(definition, LuceneIndexConstants.MERGE_POLICY_NAME, null);
+ MergePolicy mergePolicy = null;
+ if (mergePolicyName != null) {
+ if (mergePolicyName.equalsIgnoreCase("no")) {
+ mergePolicy = NoMergePolicy.COMPOUND_FILES;
+ } else if (mergePolicyName.equalsIgnoreCase("mitigated") || mergePolicyName.equalsIgnoreCase("default")) {
+ mergePolicy = new CommitMitigatingTieredMergePolicy();
+ } else if (mergePolicyName.equalsIgnoreCase("tiered")) {
+ mergePolicy = new TieredMergePolicy();
+ } else if (mergePolicyName.equalsIgnoreCase("logbyte")) {
+ mergePolicy = new LogByteSizeMergePolicy();
+ } else if (mergePolicyName.equalsIgnoreCase("logdoc")) {
+ mergePolicy = new LogDocMergePolicy();
+ }
+ }
+ if (mergePolicy == null) {
+ mergePolicy = new CommitMitigatingTieredMergePolicy();
+ }
+ return mergePolicy;
+ }
+
private static Set<String> getMultiProperty(NodeState definition, String propName){
PropertyState pse = definition.getProperty(propName);
return pse != null ? ImmutableSet.copyOf(pse.getValue(Type.STRINGS)) : Collections.<String>emptySet();
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1806013&r1=1806012&r2=1806013&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java Thu Aug 24 09:15:10 2017
@@ -114,6 +114,11 @@ public interface LuceneIndexConstants {
String CODEC_NAME = "codec";
/**
+ * Name of the merge policy to be used while indexing
+ */
+ String MERGE_POLICY_NAME = "mergePolicy";
+
+ /**
* Child node name under which property details are provided
*/
String PROP_NODE = "properties";
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java?rev=1806013&r1=1806012&r2=1806013&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java Thu Aug 24 09:15:10 2017
@@ -100,6 +100,11 @@ public final class IndexDefinitionBuilde
return this;
}
+ public IndexDefinitionBuilder mergePolicy(String mergePolicy) {
+ tree.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, checkNotNull(mergePolicy));
+ return this;
+ }
+
public IndexDefinitionBuilder noAsync(){
tree.removeProperty("async");
return this;
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java?rev=1806013&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java Thu Aug 24 09:15:10 2017
@@ -0,0 +1,661 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.writer;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentInfos;
+
+/**
+ * This {@link MergePolicy} extends Lucene's {@link org.apache.lucene.index.TieredMergePolicy} by providing mitigation
+ * to the aggressiveness of merges in case the index is under high commit load.
+ * That's because in the case of Oak we currently have that we store Lucene indexes in storage systems which require
+ * themselves some garbage collection task to be executed to get rid of deleted / unused files, similarly to what Lucene's
+ * merge does.
+ * So the bottom line is that with this {@link MergePolicy} we should have less but bigger merges, only after commit rate
+ * is under a certain threshold (in terms of added docs per sec and MBs per sec).
+ *
+ */
+public class CommitMitigatingTieredMergePolicy extends MergePolicy {
+ /** Default noCFSRatio. If a merge's size is >= 10% of
+ * the index, then we disable compound file for it.
+ * @see MergePolicy#setNoCFSRatio */
+ public static final double DEFAULT_NO_CFS_RATIO = 0.1;
+
+ public static final double DEFAULT_MAX_COMMIT_RATE_DOCS = 1000;
+ private static final double DEFAULT_MAX_COMMIT_RATE_MB = 5;
+
+ private int maxMergeAtOnce = 10;
+ private long maxMergedSegmentBytes = 5 * 1024 * 1024 * 1024L;
+ private int maxMergeAtOnceExplicit = 30;
+
+ private long floorSegmentBytes = 2 * 1024 * 1024L;
+ private double segsPerTier = 10.0;
+ private double forceMergeDeletesPctAllowed = 10.0;
+ private double reclaimDeletesWeight = 1.5;
+
+ private double maxCommitRateDocs = DEFAULT_MAX_COMMIT_RATE_DOCS;
+ private double maxCommitRateMB = DEFAULT_MAX_COMMIT_RATE_MB;
+ private double docCount = 0d;
+ private double mb = 0d;
+ private double time = System.currentTimeMillis();
+
+ /** Sole constructor, setting all settings to their
+ * defaults. */
+ public CommitMitigatingTieredMergePolicy() {
+ super(DEFAULT_NO_CFS_RATIO, MergePolicy.DEFAULT_MAX_CFS_SEGMENT_SIZE);
+ }
+
+ /** Maximum number of segments to be merged at a time
+ * during "normal" merging. For explicit merging (eg,
+ * forceMerge or forceMergeDeletes was called), see {@link
+ * #setMaxMergeAtOnceExplicit}. Default is 10. */
+ public CommitMitigatingTieredMergePolicy setMaxMergeAtOnce(int v) {
+ if (v < 2) {
+ throw new IllegalArgumentException("maxMergeAtOnce must be > 1 (got " + v + ")");
+ }
+ maxMergeAtOnce = v;
+ return this;
+ }
+
+ public CommitMitigatingTieredMergePolicy setMaxCommitRateDocs(double maxCommitRate) {
+ this.maxCommitRateDocs = maxCommitRate;
+ return this;
+ }
+
+ /** Returns the current maxMergeAtOnce setting.
+ *
+ * @see #setMaxMergeAtOnce */
+ public int getMaxMergeAtOnce() {
+ return maxMergeAtOnce;
+ }
+
+ // TODO: should addIndexes do explicit merging, too? And,
+ // if user calls IW.maybeMerge "explicitly"
+
+ /** Maximum number of segments to be merged at a time,
+ * during forceMerge or forceMergeDeletes. Default is 30. */
+ public CommitMitigatingTieredMergePolicy setMaxMergeAtOnceExplicit(int v) {
+ if (v < 2) {
+ throw new IllegalArgumentException("maxMergeAtOnceExplicit must be > 1 (got " + v + ")");
+ }
+ maxMergeAtOnceExplicit = v;
+ return this;
+ }
+
+ /** Returns the current maxMergeAtOnceExplicit setting.
+ *
+ * @see #setMaxMergeAtOnceExplicit */
+ public int getMaxMergeAtOnceExplicit() {
+ return maxMergeAtOnceExplicit;
+ }
+
+ /** Maximum sized segment to produce during
+ * normal merging. This setting is approximate: the
+ * estimate of the merged segment size is made by summing
+ * sizes of to-be-merged segments (compensating for
+ * percent deleted docs). Default is 5 GB. */
+ public CommitMitigatingTieredMergePolicy setMaxMergedSegmentMB(double v) {
+ if (v < 0.0) {
+ throw new IllegalArgumentException("maxMergedSegmentMB must be >=0 (got " + v + ")");
+ }
+ v *= 1024 * 1024;
+ maxMergedSegmentBytes = (v > Long.MAX_VALUE) ? Long.MAX_VALUE : (long) v;
+ return this;
+ }
+
+ /** Returns the current maxMergedSegmentMB setting.
+ *
+ * @see #getMaxMergedSegmentMB */
+ public double getMaxMergedSegmentMB() {
+ return maxMergedSegmentBytes / 1024 / 1024.;
+ }
+
+ /** Controls how aggressively merges that reclaim more
+ * deletions are favored. Higher values will more
+ * aggressively target merges that reclaim deletions, but
+ * be careful not to go so high that way too much merging
+ * takes place; a value of 3.0 is probably nearly too
+ * high. A value of 0.0 means deletions don't impact
+ * merge selection. */
+ public CommitMitigatingTieredMergePolicy setReclaimDeletesWeight(double v) {
+ if (v < 0.0) {
+ throw new IllegalArgumentException("reclaimDeletesWeight must be >= 0.0 (got " + v + ")");
+ }
+ reclaimDeletesWeight = v;
+ return this;
+ }
+
+ /** See {@link #setReclaimDeletesWeight}. */
+ public double getReclaimDeletesWeight() {
+ return reclaimDeletesWeight;
+ }
+
+ /** Segments smaller than this are "rounded up" to this
+ * size, ie treated as equal (floor) size for merge
+ * selection. This is to prevent frequent flushing of
+ * tiny segments from allowing a long tail in the index.
+ * Default is 2 MB. */
+ public CommitMitigatingTieredMergePolicy setFloorSegmentMB(double v) {
+ if (v <= 0.0) {
+ throw new IllegalArgumentException("floorSegmentMB must be >= 0.0 (got " + v + ")");
+ }
+ v *= 1024 * 1024;
+ floorSegmentBytes = (v > Long.MAX_VALUE) ? Long.MAX_VALUE : (long) v;
+ return this;
+ }
+
+ /** Returns the current floorSegmentMB.
+ *
+ * @see #setFloorSegmentMB */
+ public double getFloorSegmentMB() {
+ return floorSegmentBytes / (1024 * 1024.);
+ }
+
+ /** When forceMergeDeletes is called, we only merge away a
+ * segment if its delete percentage is over this
+ * threshold. Default is 10%. */
+ public CommitMitigatingTieredMergePolicy setForceMergeDeletesPctAllowed(double v) {
+ if (v < 0.0 || v > 100.0) {
+ throw new IllegalArgumentException("forceMergeDeletesPctAllowed must be between 0.0 and 100.0 inclusive (got " + v + ")");
+ }
+ forceMergeDeletesPctAllowed = v;
+ return this;
+ }
+
+ /** Returns the current forceMergeDeletesPctAllowed setting.
+ *
+ * @see #setForceMergeDeletesPctAllowed */
+ public double getForceMergeDeletesPctAllowed() {
+ return forceMergeDeletesPctAllowed;
+ }
+
+ /** Sets the allowed number of segments per tier. Smaller
+ * values mean more merging but fewer segments.
+ *
+ * <p><b>NOTE</b>: this value should be >= the {@link
+ * #setMaxMergeAtOnce} otherwise you'll force too much
+ * merging to occur.</p>
+ *
+ * <p>Default is 10.0.</p> */
+ public CommitMitigatingTieredMergePolicy setSegmentsPerTier(double v) {
+ if (v < 2.0) {
+ throw new IllegalArgumentException("segmentsPerTier must be >= 2.0 (got " + v + ")");
+ }
+ segsPerTier = v;
+ return this;
+ }
+
+ /** Returns the current segmentsPerTier setting.
+ *
+ * @see #setSegmentsPerTier */
+ public double getSegmentsPerTier() {
+ return segsPerTier;
+ }
+
+ public CommitMitigatingTieredMergePolicy setMaxCommitRateMB(int maxCommitRateMB) {
+ this.maxCommitRateMB = maxCommitRateMB;
+ return this;
+ }
+
+ private class SegmentByteSizeDescending implements Comparator<SegmentCommitInfo> {
+ @Override
+ public int compare(SegmentCommitInfo o1, SegmentCommitInfo o2) {
+ try {
+ final long sz1 = size(o1);
+ final long sz2 = size(o2);
+ if (sz1 > sz2) {
+ return -1;
+ } else if (sz2 > sz1) {
+ return 1;
+ } else {
+ return o1.info.name.compareTo(o2.info.name);
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ }
+
+ /** Holds score and explanation for a single candidate
+ * merge. */
+ protected static abstract class MergeScore {
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected MergeScore() {
+ }
+
+ /** Returns the score for this merge candidate; lower
+ * scores are better. */
+ abstract double getScore();
+
+ /** Human readable explanation of how the merge got this
+ * score. */
+ abstract String getExplanation();
+ }
+
+ @Override
+ public MergeSpecification findMerges(MergeTrigger mergeTrigger, SegmentInfos infos) throws IOException {
+ if (verbose()) {
+ message("findMerges: " + infos.size() + " segments");
+ }
+ if (infos.size() == 0) {
+ return null;
+ }
+
+ long now = System.currentTimeMillis();
+ double timeDelta = (now / 1000d) - (time / 1000d);
+ double commitRate = Math.abs(docCount - infos.totalDocCount()) / timeDelta;
+
+ docCount = infos.totalDocCount();
+ time = now;
+
+ if (verbose()) {
+ message(commitRate + "doc/s (max: " + maxCommitRateDocs + "doc/s)");
+ }
+
+ if (commitRate > maxCommitRateDocs) {
+ return null;
+ }
+
+ final Collection<SegmentCommitInfo> merging = writer.get().getMergingSegments();
+ final Collection<SegmentCommitInfo> toBeMerged = new HashSet<SegmentCommitInfo>();
+
+ final List<SegmentCommitInfo> infosSorted = new ArrayList<SegmentCommitInfo>(infos.asList());
+ Collections.sort(infosSorted, new SegmentByteSizeDescending());
+
+ // Compute total index bytes & print details about the index
+ long totIndexBytes = 0;
+ long minSegmentBytes = Long.MAX_VALUE;
+ for (SegmentCommitInfo info : infosSorted) {
+ final long segBytes = size(info);
+ if (verbose()) {
+ String extra = merging.contains(info) ? " [merging]" : "";
+ if (segBytes >= maxMergedSegmentBytes / 2.0) {
+ extra += " [skip: too large]";
+ } else if (segBytes < floorSegmentBytes) {
+ extra += " [floored]";
+ }
+ message(" seg=" + writer.get().segString(info) + " size=" + String.format(Locale.ROOT, "%.3f", segBytes / 1024 / 1024.) + " MB" + extra);
+ }
+
+ minSegmentBytes = Math.min(segBytes, minSegmentBytes);
+ // Accum total byte size
+ totIndexBytes += segBytes;
+ }
+
+ // If we have too-large segments, grace them out
+ // of the maxSegmentCount:
+ int tooBigCount = 0;
+ while (tooBigCount < infosSorted.size() && size(infosSorted.get(tooBigCount)) >= maxMergedSegmentBytes / 2.0) {
+ totIndexBytes -= size(infosSorted.get(tooBigCount));
+ tooBigCount++;
+ }
+
+ minSegmentBytes = floorSize(minSegmentBytes);
+
+ // Compute max allowed segs in the index
+ long levelSize = minSegmentBytes;
+ long bytesLeft = totIndexBytes;
+ double allowedSegCount = 0;
+ while (true) {
+ final double segCountLevel = bytesLeft / (double) levelSize;
+ if (segCountLevel < segsPerTier) {
+ allowedSegCount += Math.ceil(segCountLevel);
+ break;
+ }
+ allowedSegCount += segsPerTier;
+ bytesLeft -= segsPerTier * levelSize;
+ levelSize *= maxMergeAtOnce;
+ }
+ int allowedSegCountInt = (int) allowedSegCount;
+
+ MergeSpecification spec = null;
+
+ // Cycle to possibly select more than one merge:
+ while (true) {
+
+ long mergingBytes = 0;
+ double idxBytes = 0;
+
+ // Gather eligible segments for merging, ie segments
+ // not already being merged and not already picked (by
+ // prior iteration of this loop) for merging:
+ final List<SegmentCommitInfo> eligible = new ArrayList<SegmentCommitInfo>();
+ for (int idx = tooBigCount; idx < infosSorted.size(); idx++) {
+ final SegmentCommitInfo info = infosSorted.get(idx);
+ if (merging.contains(info)) {
+ mergingBytes += info.sizeInBytes();
+ } else if (!toBeMerged.contains(info)) {
+ eligible.add(info);
+ }
+ idxBytes += info.sizeInBytes();
+ }
+ idxBytes /= 1024 * 1000;
+
+ final boolean maxMergeIsRunning = mergingBytes >= maxMergedSegmentBytes;
+
+ if (verbose()) {
+ message(" allowedSegmentCount=" + allowedSegCountInt + " vs count=" + infosSorted.size() + " (eligible count=" + eligible.size() + ") tooBigCount=" + tooBigCount);
+ }
+
+ if (eligible.size() == 0) {
+ return spec;
+ }
+
+ double bytes = idxBytes - this.mb;
+ double mbRate = bytes / timeDelta;
+
+ if (verbose()) {
+ message(mbRate + "mb/s (max: " + maxCommitRateMB + "mb/s)");
+ }
+
+ this.mb = idxBytes;
+ if (mbRate > maxCommitRateMB) {
+ return null;
+ }
+
+ if (eligible.size() >= allowedSegCountInt) {
+
+ // OK we are over budget -- find best merge!
+ MergeScore bestScore = null;
+ List<SegmentCommitInfo> best = null;
+ boolean bestTooLarge = false;
+ long bestMergeBytes = 0;
+
+ // Consider all merge starts:
+ for (int startIdx = 0; startIdx <= eligible.size() - maxMergeAtOnce; startIdx++) {
+
+ long totAfterMergeBytes = 0;
+
+ final List<SegmentCommitInfo> candidate = new ArrayList<SegmentCommitInfo>();
+ boolean hitTooLarge = false;
+ for (int idx = startIdx; idx < eligible.size() && candidate.size() < maxMergeAtOnce; idx++) {
+ final SegmentCommitInfo info = eligible.get(idx);
+ final long segBytes = size(info);
+
+ if (totAfterMergeBytes + segBytes > maxMergedSegmentBytes) {
+ hitTooLarge = true;
+ // NOTE: we continue, so that we can try
+ // "packing" smaller segments into this merge
+ // to see if we can get closer to the max
+ // size; this in general is not perfect since
+ // this is really "bin packing" and we'd have
+ // to try different permutations.
+ continue;
+ }
+ candidate.add(info);
+ totAfterMergeBytes += segBytes;
+ }
+
+ final MergeScore score = score(candidate, hitTooLarge, mergingBytes);
+ if (verbose()) {
+ message(" maybe=" + writer.get().segString(candidate) + " score=" + score.getScore() + " " + score.getExplanation() + " tooLarge=" + hitTooLarge + " size=" + String.format(Locale.ROOT, "%.3f MB", totAfterMergeBytes / 1024. / 1024.));
+ }
+
+ // If we are already running a max sized merge
+ // (maxMergeIsRunning), don't allow another max
+ // sized merge to kick off:
+ if ((bestScore == null || score.getScore() < bestScore.getScore()) && (!hitTooLarge || !maxMergeIsRunning)) {
+ best = candidate;
+ bestScore = score;
+ bestTooLarge = hitTooLarge;
+ bestMergeBytes = totAfterMergeBytes;
+ }
+ }
+
+ if (best != null) {
+ if (spec == null) {
+ spec = new MergeSpecification();
+ }
+ final OneMerge merge = new OneMerge(best);
+ spec.add(merge);
+ for (SegmentCommitInfo info : merge.segments) {
+ toBeMerged.add(info);
+ }
+ if (verbose()) {
+ message(" add merge=" + writer.get().segString(merge.segments) + " size=" + String.format(Locale.ROOT, "%.3f MB", bestMergeBytes / 1024. / 1024.) + " score=" + String.format(Locale.ROOT, "%.3f", bestScore.getScore()) + " " + bestScore.getExplanation() + (bestTooLarge ? " [max merge]" : ""));
+ }
+ } else {
+ return spec;
+ }
+ } else {
+ return spec;
+ }
+ }
+ }
+
+ /** Expert: scores one merge; subclasses can override. */
+ protected MergeScore score(List<SegmentCommitInfo> candidate, boolean hitTooLarge, long mergingBytes) throws IOException {
+ long totBeforeMergeBytes = 0;
+ long totAfterMergeBytes = 0;
+ long totAfterMergeBytesFloored = 0;
+ for (SegmentCommitInfo info : candidate) {
+ final long segBytes = size(info);
+ totAfterMergeBytes += segBytes;
+ totAfterMergeBytesFloored += floorSize(segBytes);
+ totBeforeMergeBytes += info.sizeInBytes();
+ }
+
+ // Roughly measure "skew" of the merge, i.e. how
+ // "balanced" the merge is (whether the segments are
+ // about the same size), which can range from
+ // 1.0/numSegsBeingMerged (good) to 1.0 (poor). Heavily
+ // lopsided merges (skew near 1.0) is no good; it means
+ // O(N^2) merge cost over time:
+ final double skew;
+ if (hitTooLarge) {
+ // Pretend the merge has perfect skew; skew doesn't
+ // matter in this case because this merge will not
+ // "cascade" and so it cannot lead to N^2 merge cost
+ // over time:
+ skew = 1.0 / maxMergeAtOnce;
+ } else {
+ skew = ((double) floorSize(size(candidate.get(0)))) / totAfterMergeBytesFloored;
+ }
+
+ // Strongly favor merges with less skew (smaller
+ // mergeScore is better):
+ double mergeScore = skew;
+
+ // Gently favor smaller merges over bigger ones. We
+ // don't want to make this exponent too large else we
+ // can end up doing poor merges of small segments in
+ // order to avoid the large merges:
+ mergeScore *= Math.pow(totAfterMergeBytes, 0.05);
+
+ // Strongly favor merges that reclaim deletes:
+ final double nonDelRatio = ((double) totAfterMergeBytes) / totBeforeMergeBytes;
+ mergeScore *= Math.pow(nonDelRatio, reclaimDeletesWeight);
+
+ final double finalMergeScore = mergeScore;
+
+ return new MergeScore() {
+
+ @Override
+ public double getScore() {
+ return finalMergeScore;
+ }
+
+ @Override
+ public String getExplanation() {
+ return "skew=" + String.format(Locale.ROOT, "%.3f", skew) + " nonDelRatio=" + String.format(Locale.ROOT, "%.3f", nonDelRatio);
+ }
+ };
+ }
+
+ @Override
+ public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCount, Map<SegmentCommitInfo, Boolean> segmentsToMerge) throws IOException {
+ if (verbose()) {
+ message("findForcedMerges maxSegmentCount=" + maxSegmentCount + " infos=" + writer.get().segString(infos) + " segmentsToMerge=" + segmentsToMerge);
+ }
+
+ List<SegmentCommitInfo> eligible = new ArrayList<SegmentCommitInfo>();
+ boolean forceMergeRunning = false;
+ final Collection<SegmentCommitInfo> merging = writer.get().getMergingSegments();
+ boolean segmentIsOriginal = false;
+ for (SegmentCommitInfo info : infos) {
+ final Boolean isOriginal = segmentsToMerge.get(info);
+ if (isOriginal != null) {
+ segmentIsOriginal = isOriginal;
+ if (!merging.contains(info)) {
+ eligible.add(info);
+ } else {
+ forceMergeRunning = true;
+ }
+ }
+ }
+
+ if (eligible.size() == 0) {
+ return null;
+ }
+
+ if ((maxSegmentCount > 1 && eligible.size() <= maxSegmentCount) ||
+ (maxSegmentCount == 1 && eligible.size() == 1 && (!segmentIsOriginal || isMerged(infos, eligible.get(0))))) {
+ if (verbose()) {
+ message("already merged");
+ }
+ return null;
+ }
+
+ Collections.sort(eligible, new SegmentByteSizeDescending());
+
+ if (verbose()) {
+ message("eligible=" + eligible);
+ message("forceMergeRunning=" + forceMergeRunning);
+ }
+
+ int end = eligible.size();
+
+ MergeSpecification spec = null;
+
+ // Do full merges, first, backwards:
+ while (end >= maxMergeAtOnceExplicit + maxSegmentCount - 1) {
+ if (spec == null) {
+ spec = new MergeSpecification();
+ }
+ final OneMerge merge = new OneMerge(eligible.subList(end - maxMergeAtOnceExplicit, end));
+ if (verbose()) {
+ message("add merge=" + writer.get().segString(merge.segments));
+ }
+ spec.add(merge);
+ end -= maxMergeAtOnceExplicit;
+ }
+
+ if (spec == null && !forceMergeRunning) {
+ // Do final merge
+ final int numToMerge = end - maxSegmentCount + 1;
+ final OneMerge merge = new OneMerge(eligible.subList(end - numToMerge, end));
+ if (verbose()) {
+ message("add final merge=" + merge.segString(writer.get().getDirectory()));
+ }
+ spec = new MergeSpecification();
+ spec.add(merge);
+ }
+
+ return spec;
+ }
+
+ @Override
+ public MergeSpecification findForcedDeletesMerges(SegmentInfos infos) throws IOException {
+ if (verbose()) {
+ message("findForcedDeletesMerges infos=" + writer.get().segString(infos) + " forceMergeDeletesPctAllowed=" + forceMergeDeletesPctAllowed);
+ }
+ final List<SegmentCommitInfo> eligible = new ArrayList<SegmentCommitInfo>();
+ final Collection<SegmentCommitInfo> merging = writer.get().getMergingSegments();
+ for (SegmentCommitInfo info : infos) {
+ double pctDeletes = 100. * ((double) writer.get().numDeletedDocs(info)) / info.info.getDocCount();
+ if (pctDeletes > forceMergeDeletesPctAllowed && !merging.contains(info)) {
+ eligible.add(info);
+ }
+ }
+
+ if (eligible.size() == 0) {
+ return null;
+ }
+
+ Collections.sort(eligible, new SegmentByteSizeDescending());
+
+ if (verbose()) {
+ message("eligible=" + eligible);
+ }
+
+ int start = 0;
+ MergeSpecification spec = null;
+
+ while (start < eligible.size()) {
+ // Don't enforce max merged size here: app is explicitly
+ // calling forceMergeDeletes, and knows this may take a
+ // long time / produce big segments (like forceMerge):
+ final int end = Math.min(start + maxMergeAtOnceExplicit, eligible.size());
+ if (spec == null) {
+ spec = new MergeSpecification();
+ }
+
+ final OneMerge merge = new OneMerge(eligible.subList(start, end));
+ if (verbose()) {
+ message("add merge=" + writer.get().segString(merge.segments));
+ }
+ spec.add(merge);
+ start = end;
+ }
+
+ return spec;
+ }
+
+ @Override
+ public void close() {
+ }
+
+ private long floorSize(long bytes) {
+ return Math.max(floorSegmentBytes, bytes);
+ }
+
+ private boolean verbose() {
+ final IndexWriter w = writer.get();
+ return w != null && w.getConfig().getInfoStream().isEnabled("TMP");
+ }
+
+ private void message(String message) {
+ writer.get().getConfig().getInfoStream().message("TMP", message);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("[" + getClass().getSimpleName() + ": ");
+ sb.append("maxMergeAtOnce=").append(maxMergeAtOnce).append(", ");
+ sb.append("maxMergeAtOnceExplicit=").append(maxMergeAtOnceExplicit).append(", ");
+ sb.append("maxMergedSegmentMB=").append(maxMergedSegmentBytes / 1024 / 1024.).append(", ");
+ sb.append("floorSegmentMB=").append(floorSegmentBytes / 1024 / 1024.).append(", ");
+ sb.append("forceMergeDeletesPctAllowed=").append(forceMergeDeletesPctAllowed).append(", ");
+ sb.append("segmentsPerTier=").append(segsPerTier).append(", ");
+ sb.append("maxCFSSegmentSizeMB=").append(getMaxCFSSegmentSizeMB()).append(", ");
+ sb.append("noCFSRatio=").append(noCFSRatio);
+ return sb.toString();
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java?rev=1806013&r1=1806012&r2=1806013&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java Thu Aug 24 09:15:10 2017
@@ -148,6 +148,7 @@ class DefaultIndexWriter implements Luce
} else {
config = getIndexWriterConfig(definition, true);
}
+ config.setMergePolicy(definition.getMergePolicy());
writer = new IndexWriter(directory, config);
genAtStart = getLatestGeneration(directory);
PERF_LOGGER.end(start, -1, "Created IndexWriter for directory {}", definition);
@@ -159,7 +160,6 @@ class DefaultIndexWriter implements Luce
* eventually update suggest dictionary
* @throws IOException if suggest dictionary update fails
* @param analyzer the analyzer used to update the suggester
- * @param blobStore
*/
private boolean updateSuggester(Analyzer analyzer, Calendar currentTime) throws IOException {
NodeBuilder suggesterStatus = definitionBuilder.child(suggestDirName);
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java?rev=1806013&r1=1806012&r2=1806013&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java Thu Aug 24 09:15:10 2017
@@ -51,6 +51,7 @@ public class IndexWriterUtils {
}
Analyzer analyzer = new PerFieldAnalyzerWrapper(definitionAnalyzer, analyzers);
IndexWriterConfig config = new IndexWriterConfig(VERSION, analyzer);
+// config.setMergePolicy(definition.getMergePolicy());
if (remoteDir) {
config.setMergeScheduler(new SerialMergeScheduler());
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java?rev=1806013&r1=1806012&r2=1806013&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java Thu Aug 24 09:15:10 2017
@@ -30,9 +30,14 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.IndexingMode;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.TokenizerChain;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.CommitMitigatingTieredMergePolicy;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.LogByteSizeMergePolicy;
+import org.apache.lucene.index.LogDocMergePolicy;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
import org.junit.Test;
import static com.google.common.collect.ImmutableSet.of;
@@ -154,6 +159,49 @@ public class IndexDefinitionTest {
}
@Test
+ public void mergePolicyConfig() throws Exception{
+ IndexDefinition defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+ assertNotNull(defn.getMergePolicy());
+ assertEquals(CommitMitigatingTieredMergePolicy.class, defn.getMergePolicy().getClass());
+
+ builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "tiered");
+ defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+ assertNotNull(defn.getMergePolicy());
+ assertEquals(TieredMergePolicy.class, defn.getMergePolicy().getClass());
+
+ builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "logbyte");
+ defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+ assertNotNull(defn.getMergePolicy());
+ assertEquals(LogByteSizeMergePolicy.class, defn.getMergePolicy().getClass());
+
+ builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "logdoc");
+ defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+ assertNotNull(defn.getMergePolicy());
+ assertEquals(LogDocMergePolicy.class, defn.getMergePolicy().getClass());
+
+ builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "no");
+ defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+ assertNotNull(defn.getMergePolicy());
+ assertEquals(NoMergePolicy.class, defn.getMergePolicy().getClass());
+
+ builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "default");
+ defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+ assertNotNull(defn.getMergePolicy());
+ assertEquals(CommitMitigatingTieredMergePolicy.class, defn.getMergePolicy().getClass());
+
+ builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "mitigated");
+ defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+ assertNotNull(defn.getMergePolicy());
+ assertEquals(CommitMitigatingTieredMergePolicy.class, defn.getMergePolicy().getClass());
+
+ // wrong mp name falls back to default
+ builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "whoawhoa");
+ defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+ assertNotNull(defn.getMergePolicy());
+ assertEquals(CommitMitigatingTieredMergePolicy.class, defn.getMergePolicy().getClass());
+ }
+
+ @Test
public void relativePropertyConfig() throws Exception{
builder.child(PROP_NODE).child("foo1").child("bar").setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_DATE);
builder.child(PROP_NODE).child("foo2").child("bar2").child("baz").setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_LONG);
Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java?rev=1806013&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java Thu Aug 24 09:15:10 2017
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+
+import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.core.data.FileDataStore;
+import org.apache.jackrabbit.oak.InitialContent;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.api.ContentRepository;
+import org.apache.jackrabbit.oak.api.Tree;
+import org.apache.jackrabbit.oak.commons.IOUtils;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.plugins.blob.BlobStoreStats;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.CopyOnReadDirectory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+import org.apache.jackrabbit.oak.query.AbstractQueryTest;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
+import org.apache.jackrabbit.oak.segment.file.FileStoreStats;
+import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.blob.stats.BlobStatsCollector;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.FilterDirectory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/**
+ * Tests for checking impacts of Lucene writes wrt storage / configuration adjustments on the
+ * {@link org.apache.jackrabbit.oak.segment.SegmentNodeStore}.
+ */
+@Ignore("this is meant to be a benchmark, it shouldn't be part of everyday builds")
+@RunWith(Parameterized.class)
+public class LuceneWritesOnSegmentStatsTest extends AbstractQueryTest {
+
+ private static final File DIRECTORY = new File("target/fs");
+
+ private final boolean copyOnRW;
+ private final String codec;
+ private final boolean indexOnFS;
+ private final int minRecordLength;
+ private final String mergePolicy;
+
+ private ExecutorService executorService = Executors.newFixedThreadPool(2);
+
+ private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target"));
+
+ private String corDir = null;
+ private String cowDir = null;
+
+ private TestUtil.OptionalEditorProvider optionalEditorProvider = new TestUtil.OptionalEditorProvider();
+
+ private FileStore fileStore;
+ private DataStoreBlobStore dataStoreBlobStore;
+ private DefaultStatisticsProvider statisticsProvider;
+ private String fdsDir;
+ private String indexPath;
+
+
+ public LuceneWritesOnSegmentStatsTest(boolean copyOnRW, String codec, boolean indexOnFS, int minRecordLength, String mergePolicy) {
+ this.copyOnRW = copyOnRW;
+ this.codec = codec;
+ this.indexOnFS = indexOnFS;
+ this.minRecordLength = minRecordLength;
+ this.mergePolicy = mergePolicy;
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][]{
+ {false, "oakCodec", false, 4000, "tiered"},
+ {false, "oakCodec", false, 4000, "mitigated"},
+ {false, "oakCodec", false, 4000, "no"},
+ {false, "Lucene46", false, 4000, "tiered"},
+ {false, "Lucene46", false, 4000, "mitigated"},
+ {false, "Lucene46", false, 4000, "no"},
+ {false, "oakCodec", false, -1, "tiered"},
+ {false, "oakCodec", false, -1, "mitigated"},
+ {false, "oakCodec", false, -1, "no"},
+ {false, "Lucene46", false, -1, "tiered"},
+ {false, "Lucene46", false, -1, "mitigated"},
+ {false, "Lucene46", false, -1, "no"},
+ });
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ if (!DIRECTORY.exists()) {
+ assert DIRECTORY.mkdirs();
+ }
+ }
+
+ @After
+ public void after() {
+ new ExecutorCloser(executorService).close();
+ IndexDefinition.setDisableStoredIndexDefinition(false);
+ fileStore.close();
+ if (DIRECTORY.exists()) {
+ try {
+ FileUtils.deleteDirectory(DIRECTORY);
+ } catch (IOException e) {
+ // do nothing
+ }
+ }
+ }
+
+ @Override
+ protected void createTestIndexNode() throws Exception {
+ setTraversalEnabled(false);
+ }
+
+ @Override
+ protected ContentRepository createRepository() {
+ LuceneIndexEditorProvider editorProvider;
+ LuceneIndexProvider provider;
+ if (copyOnRW) {
+ IndexCopier copier = createIndexCopier();
+ editorProvider = new LuceneIndexEditorProvider(copier, new ExtractedTextCache(10 * FileUtils.ONE_MB, 100));
+ provider = new LuceneIndexProvider(copier);
+ } else {
+ editorProvider = new LuceneIndexEditorProvider();
+ provider = new LuceneIndexProvider();
+ }
+
+ NodeStore nodeStore;
+ try {
+ statisticsProvider = new DefaultStatisticsProvider(scheduledExecutorService);
+ fileStore = FileStoreBuilder.fileStoreBuilder(DIRECTORY)
+ .withStatisticsProvider(statisticsProvider)
+ .withBlobStore(createBlobStore())
+ .build();
+ nodeStore = SegmentNodeStoreBuilders.builder(fileStore).build();
+ } catch (IOException | InvalidFileStoreVersionException e) {
+ throw new RuntimeException(e);
+ }
+ return new Oak(nodeStore)
+ .with(new InitialContent())
+ .with(new OpenSecurityProvider())
+ .with((QueryIndexProvider) provider)
+ .with((Observer) provider)
+ .with(editorProvider)
+ .with(optionalEditorProvider)
+ .with(new PropertyIndexEditorProvider())
+ .with(new NodeTypeIndexProvider())
+ .createContentRepository();
+ }
+
+ private BlobStore createBlobStore() {
+ FileDataStore fds = new OakFileDataStore();
+ fdsDir = "target/fds-" + codec + copyOnRW + minRecordLength + mergePolicy;
+ fds.setPath(fdsDir);
+ if (minRecordLength > 0) {
+ fds.setMinRecordLength(minRecordLength);
+ }
+ fds.init(null);
+ dataStoreBlobStore = new DataStoreBlobStore(fds);
+ StatisticsProvider sp = new DefaultStatisticsProvider(scheduledExecutorService);
+ BlobStatsCollector collector = new BlobStoreStats(sp);
+ dataStoreBlobStore.setBlobStatsCollector(collector);
+ return dataStoreBlobStore;
+ }
+
+ private IndexCopier createIndexCopier() {
+ try {
+ return new IndexCopier(executorService, temporaryFolder.getRoot()) {
+ @Override
+ public Directory wrapForRead(String indexPath, IndexDefinition definition,
+ Directory remote, String dirName) throws IOException {
+ Directory ret = super.wrapForRead(indexPath, definition, remote, dirName);
+ corDir = getFSDirPath(ret);
+ return ret;
+ }
+
+ @Override
+ public Directory wrapForWrite(IndexDefinition definition,
+ Directory remote, boolean reindexMode, String dirName) throws IOException {
+ Directory ret = super.wrapForWrite(definition, remote, reindexMode, dirName);
+ cowDir = getFSDirPath(ret);
+ return ret;
+ }
+
+ private String getFSDirPath(Directory dir) {
+ if (dir instanceof CopyOnReadDirectory) {
+ dir = ((CopyOnReadDirectory) dir).getLocal();
+ }
+
+ dir = unwrap(dir);
+
+ if (dir instanceof FSDirectory) {
+ return ((FSDirectory) dir).getDirectory().getAbsolutePath();
+ }
+ return null;
+ }
+
+ private Directory unwrap(Directory dir) {
+ if (dir instanceof FilterDirectory) {
+ return unwrap(((FilterDirectory) dir).getDelegate());
+ }
+ return dir;
+ }
+
+ };
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @After
+ public void shutdownExecutor() {
+ executorService.shutdown();
+ scheduledExecutorService.shutdown();
+ }
+
+ @Test
+ public void testLuceneIndexSegmentStats() throws Exception {
+ IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync().codec(codec).mergePolicy(mergePolicy);
+ idxb.indexRule("nt:base").property("foo").analyzed().nodeScopeIndex().ordered().useInExcerpt().propertyIndex();
+ idxb.indexRule("nt:base").property("bin").analyzed().nodeScopeIndex().ordered().useInExcerpt().propertyIndex();
+ Tree idx = root.getTree("/").getChild("oak:index").addChild("lucenePropertyIndex");
+ Tree idxDef = idxb.build(idx);
+ if (!codec.equals("oakCodec") && indexOnFS) {
+ idxDef.setProperty("persistence", "file");
+ indexPath = "target/index-" + codec + copyOnRW;
+ idxDef.setProperty("path", indexPath);
+ }
+
+ Random r = new Random();
+
+ System.out.println("***");
+ System.out.println(codec + "," + copyOnRW + "," + indexOnFS + "," + minRecordLength + "," + mergePolicy);
+
+ long start = System.currentTimeMillis();
+ int multiplier = 3;
+ for (int n = 0; n < multiplier; n++) {
+ System.err.println("iteration " + (n + 1));
+
+ Tree rootTree = root.getTree("/").addChild("content");
+ byte[] bytes = new byte[10240];
+ Charset charset = Charset.defaultCharset();
+ for (int i = 0; i < 1000; i++) {
+ r.nextBytes(bytes);
+ String text = new String(bytes, charset);
+ Tree tree = rootTree.addChild(String.valueOf(n + i));
+ tree.setProperty("foo", text);
+ tree.setProperty("bin", bytes);
+ }
+ root.commit();
+
+ printStats();
+
+ System.out.println("reindex");
+
+ // do nothing, reindex and measure
+ idx = root.getTree("/oak:index/lucenePropertyIndex");
+ idx.setProperty("reindex", true);
+ root.commit();
+
+ printStats();
+
+ System.out.println("add and delete");
+
+ // add and delete some content and measure
+ for (int i = 0; i < 1000; i++) {
+ r.nextBytes(bytes);
+ String text = new String(bytes, charset);
+ Tree tree = rootTree.addChild(String.valueOf(n + 100 + i));
+ tree.setProperty("foo", text);
+ tree.setProperty("bin", bytes);
+ if (n + i % 3 == 0) { // delete one of the already existing nodes every 3
+ assert rootTree.getChild(String.valueOf(n + i)).remove();
+ }
+ }
+ root.commit();
+
+ printStats();
+ }
+ long time = System.currentTimeMillis() - start;
+ System.out.println("finished in " + (time / (60000)) + " minutes");
+ System.out.println("***");
+ }
+
+ private void printStats() throws IOException {
+ fileStore.flush();
+
+ FileStoreStats stats = fileStore.getStats();
+
+ long sizeOfDirectory = FileUtils.sizeOfDirectory(new File(fdsDir));
+ String fdsSize = (sizeOfDirectory / (1024 * 1000)) + " MB";
+
+ System.err.println("||codec||min record length||merge policy||segment size||FDS size||");
+ System.err.println("|"+codec+"|"+minRecordLength+"|"+mergePolicy+"|"+ IOUtils.humanReadableByteCount(stats.getApproximateSize())+"|"+fdsSize+"|");
+
+ if (indexOnFS) {
+ long sizeOfFSIndex = FileUtils.sizeOfDirectory(new File(indexPath));
+ System.out.println("Index on FS size : " + FileUtils.byteCountToDisplaySize(sizeOfFSIndex));
+ }
+ }
+
+ private long dumpFileStoreTo(File to) throws IOException {
+ if (!to.exists()) {
+ assert to.mkdirs();
+ }
+ for (File f : DIRECTORY.listFiles()) {
+ Files.copy(f, new File(to.getPath(), f.getName()));
+ }
+
+ long sizeOfDirectory = FileUtils.sizeOfDirectory(to);
+
+ to.deleteOnExit();
+ return sizeOfDirectory;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java
------------------------------------------------------------------------------
svn:eol-style = native