You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2017/08/24 09:15:10 UTC

svn commit: r1806013 - in /jackrabbit/oak/trunk/oak-lucene/src: main/java/org/apache/jackrabbit/oak/plugins/index/lucene/ main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/ main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/ te...

Author: tommaso
Date: Thu Aug 24 09:15:10 2017
New Revision: 1806013

URL: http://svn.apache.org/viewvc?rev=1806013&view=rev
Log:
OAK-5192, OAK-6514 - allow configuring mp, default to 'migitate' mp

Added:
    jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java   (with props)
    jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java   (with props)
Modified:
    jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
    jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
    jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java
    jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java
    jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java
    jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java

Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1806013&r1=1806012&r2=1806013&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java Thu Aug 24 09:15:10 2017
@@ -52,6 +52,7 @@ import org.apache.jackrabbit.oak.plugins
 import org.apache.jackrabbit.oak.plugins.index.lucene.util.ConfigUtil;
 import org.apache.jackrabbit.oak.plugins.index.lucene.util.FunctionIndexProcessor;
 import org.apache.jackrabbit.oak.plugins.index.lucene.util.TokenizerChain;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.CommitMitigatingTieredMergePolicy;
 import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
 import org.apache.jackrabbit.oak.plugins.nodetype.ReadOnlyNodeTypeManager;
 import org.apache.jackrabbit.oak.plugins.tree.TreeFactory;
@@ -67,6 +68,11 @@ import org.apache.lucene.analysis.miscel
 import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
 import org.apache.lucene.analysis.path.PathHierarchyTokenizerFactory;
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.LogByteSizeMergePolicy;
+import org.apache.lucene.index.LogDocMergePolicy;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -424,6 +430,12 @@ public final class IndexDefinition imple
         return codec;
     }
 
+    @Nonnull
+    public MergePolicy getMergePolicy() {
+        // MP is not cached to avoid complaining about multiple IWs with multiplexing writers
+        return createMergePolicy();
+    }
+
     public long getReindexCount(){
         return reindexCount;
     }
@@ -1494,6 +1506,28 @@ public final class IndexDefinition imple
         return codec;
     }
 
+    private MergePolicy createMergePolicy() {
+        String mergePolicyName = getOptionalValue(definition, LuceneIndexConstants.MERGE_POLICY_NAME, null);
+        MergePolicy mergePolicy = null;
+        if (mergePolicyName != null) {
+            if (mergePolicyName.equalsIgnoreCase("no")) {
+                mergePolicy = NoMergePolicy.COMPOUND_FILES;
+            } else if (mergePolicyName.equalsIgnoreCase("mitigated") || mergePolicyName.equalsIgnoreCase("default")) {
+                mergePolicy = new CommitMitigatingTieredMergePolicy();
+            } else if (mergePolicyName.equalsIgnoreCase("tiered")) {
+                mergePolicy = new TieredMergePolicy();
+            } else if (mergePolicyName.equalsIgnoreCase("logbyte")) {
+                mergePolicy = new LogByteSizeMergePolicy();
+            } else if (mergePolicyName.equalsIgnoreCase("logdoc")) {
+                mergePolicy = new LogDocMergePolicy();
+            }
+        }
+        if (mergePolicy == null) {
+            mergePolicy = new CommitMitigatingTieredMergePolicy();
+        }
+        return mergePolicy;
+    }
+
     private static Set<String> getMultiProperty(NodeState definition, String propName){
         PropertyState pse = definition.getProperty(propName);
         return pse != null ? ImmutableSet.copyOf(pse.getValue(Type.STRINGS)) : Collections.<String>emptySet();

Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1806013&r1=1806012&r2=1806013&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java Thu Aug 24 09:15:10 2017
@@ -114,6 +114,11 @@ public interface LuceneIndexConstants {
     String CODEC_NAME = "codec";
 
     /**
+     * Name of the merge policy to be used while indexing
+     */
+    String MERGE_POLICY_NAME = "mergePolicy";
+
+    /**
      * Child node name under which property details are provided
      */
     String PROP_NODE = "properties";

Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java?rev=1806013&r1=1806012&r2=1806013&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java Thu Aug 24 09:15:10 2017
@@ -100,6 +100,11 @@ public final class IndexDefinitionBuilde
         return this;
     }
 
+    public IndexDefinitionBuilder mergePolicy(String mergePolicy) {
+        tree.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, checkNotNull(mergePolicy));
+        return this;
+    }
+
     public IndexDefinitionBuilder noAsync(){
         tree.removeProperty("async");
         return this;

Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java?rev=1806013&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java Thu Aug 24 09:15:10 2017
@@ -0,0 +1,661 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.writer;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentInfos;
+
+/**
+ *  This {@link MergePolicy} extends Lucene's {@link org.apache.lucene.index.TieredMergePolicy} by providing mitigation
+ *  to the aggressiveness of merges in case the index is under high commit load.
+ *  That's because in the case of Oak we currently have that we store Lucene indexes in storage systems which require
+ *  themselves some garbage collection task to be executed to get rid of deleted / unused files, similarly to what Lucene's
+ *  merge does.
+ *  So the bottom line is that with this {@link MergePolicy} we should have less but bigger merges, only after commit rate
+ *  is under a certain threshold (in terms of added docs per sec and MBs per sec).
+ *
+ */
+public class CommitMitigatingTieredMergePolicy extends MergePolicy {
+    /** Default noCFSRatio.  If a merge's size is >= 10% of
+     *  the index, then we disable compound file for it.
+     *  @see MergePolicy#setNoCFSRatio */
+    public static final double DEFAULT_NO_CFS_RATIO = 0.1;
+
+    public static final double DEFAULT_MAX_COMMIT_RATE_DOCS = 1000;
+    private static final double DEFAULT_MAX_COMMIT_RATE_MB = 5;
+
+    private int maxMergeAtOnce = 10;
+    private long maxMergedSegmentBytes = 5 * 1024 * 1024 * 1024L;
+    private int maxMergeAtOnceExplicit = 30;
+
+    private long floorSegmentBytes = 2 * 1024 * 1024L;
+    private double segsPerTier = 10.0;
+    private double forceMergeDeletesPctAllowed = 10.0;
+    private double reclaimDeletesWeight = 1.5;
+
+    private double maxCommitRateDocs = DEFAULT_MAX_COMMIT_RATE_DOCS;
+    private double maxCommitRateMB = DEFAULT_MAX_COMMIT_RATE_MB;
+    private double docCount = 0d;
+    private double mb = 0d;
+    private double time = System.currentTimeMillis();
+
+    /** Sole constructor, setting all settings to their
+     *  defaults. */
+    public CommitMitigatingTieredMergePolicy() {
+        super(DEFAULT_NO_CFS_RATIO, MergePolicy.DEFAULT_MAX_CFS_SEGMENT_SIZE);
+    }
+
+    /** Maximum number of segments to be merged at a time
+     *  during "normal" merging.  For explicit merging (eg,
+     *  forceMerge or forceMergeDeletes was called), see {@link
+     *  #setMaxMergeAtOnceExplicit}.  Default is 10. */
+    public CommitMitigatingTieredMergePolicy setMaxMergeAtOnce(int v) {
+        if (v < 2) {
+            throw new IllegalArgumentException("maxMergeAtOnce must be > 1 (got " + v + ")");
+        }
+        maxMergeAtOnce = v;
+        return this;
+    }
+
+    public CommitMitigatingTieredMergePolicy setMaxCommitRateDocs(double maxCommitRate) {
+        this.maxCommitRateDocs = maxCommitRate;
+        return this;
+    }
+
+    /** Returns the current maxMergeAtOnce setting.
+     *
+     * @see #setMaxMergeAtOnce */
+    public int getMaxMergeAtOnce() {
+        return maxMergeAtOnce;
+    }
+
+    // TODO: should addIndexes do explicit merging, too?  And,
+    // if user calls IW.maybeMerge "explicitly"
+
+    /** Maximum number of segments to be merged at a time,
+     *  during forceMerge or forceMergeDeletes. Default is 30. */
+    public CommitMitigatingTieredMergePolicy setMaxMergeAtOnceExplicit(int v) {
+        if (v < 2) {
+            throw new IllegalArgumentException("maxMergeAtOnceExplicit must be > 1 (got " + v + ")");
+        }
+        maxMergeAtOnceExplicit = v;
+        return this;
+    }
+
+    /** Returns the current maxMergeAtOnceExplicit setting.
+     *
+     * @see #setMaxMergeAtOnceExplicit */
+    public int getMaxMergeAtOnceExplicit() {
+        return maxMergeAtOnceExplicit;
+    }
+
+    /** Maximum sized segment to produce during
+     *  normal merging.  This setting is approximate: the
+     *  estimate of the merged segment size is made by summing
+     *  sizes of to-be-merged segments (compensating for
+     *  percent deleted docs).  Default is 5 GB. */
+    public CommitMitigatingTieredMergePolicy setMaxMergedSegmentMB(double v) {
+        if (v < 0.0) {
+            throw new IllegalArgumentException("maxMergedSegmentMB must be >=0 (got " + v + ")");
+        }
+        v *= 1024 * 1024;
+        maxMergedSegmentBytes = (v > Long.MAX_VALUE) ? Long.MAX_VALUE : (long) v;
+        return this;
+    }
+
+    /** Returns the current maxMergedSegmentMB setting.
+     *
+     * @see #getMaxMergedSegmentMB */
+    public double getMaxMergedSegmentMB() {
+        return maxMergedSegmentBytes / 1024 / 1024.;
+    }
+
+    /** Controls how aggressively merges that reclaim more
+     *  deletions are favored.  Higher values will more
+     *  aggressively target merges that reclaim deletions, but
+     *  be careful not to go so high that way too much merging
+     *  takes place; a value of 3.0 is probably nearly too
+     *  high.  A value of 0.0 means deletions don't impact
+     *  merge selection. */
+    public CommitMitigatingTieredMergePolicy setReclaimDeletesWeight(double v) {
+        if (v < 0.0) {
+            throw new IllegalArgumentException("reclaimDeletesWeight must be >= 0.0 (got " + v + ")");
+        }
+        reclaimDeletesWeight = v;
+        return this;
+    }
+
+    /** See {@link #setReclaimDeletesWeight}. */
+    public double getReclaimDeletesWeight() {
+        return reclaimDeletesWeight;
+    }
+
+    /** Segments smaller than this are "rounded up" to this
+     *  size, ie treated as equal (floor) size for merge
+     *  selection.  This is to prevent frequent flushing of
+     *  tiny segments from allowing a long tail in the index.
+     *  Default is 2 MB. */
+    public CommitMitigatingTieredMergePolicy setFloorSegmentMB(double v) {
+        if (v <= 0.0) {
+            throw new IllegalArgumentException("floorSegmentMB must be >= 0.0 (got " + v + ")");
+        }
+        v *= 1024 * 1024;
+        floorSegmentBytes = (v > Long.MAX_VALUE) ? Long.MAX_VALUE : (long) v;
+        return this;
+    }
+
+    /** Returns the current floorSegmentMB.
+     *
+     *  @see #setFloorSegmentMB */
+    public double getFloorSegmentMB() {
+        return floorSegmentBytes / (1024 * 1024.);
+    }
+
+    /** When forceMergeDeletes is called, we only merge away a
+     *  segment if its delete percentage is over this
+     *  threshold.  Default is 10%. */
+    public CommitMitigatingTieredMergePolicy setForceMergeDeletesPctAllowed(double v) {
+        if (v < 0.0 || v > 100.0) {
+            throw new IllegalArgumentException("forceMergeDeletesPctAllowed must be between 0.0 and 100.0 inclusive (got " + v + ")");
+        }
+        forceMergeDeletesPctAllowed = v;
+        return this;
+    }
+
+    /** Returns the current forceMergeDeletesPctAllowed setting.
+     *
+     * @see #setForceMergeDeletesPctAllowed */
+    public double getForceMergeDeletesPctAllowed() {
+        return forceMergeDeletesPctAllowed;
+    }
+
+    /** Sets the allowed number of segments per tier.  Smaller
+     *  values mean more merging but fewer segments.
+     *
+     *  <p><b>NOTE</b>: this value should be >= the {@link
+     *  #setMaxMergeAtOnce} otherwise you'll force too much
+     *  merging to occur.</p>
+     *
+     *  <p>Default is 10.0.</p> */
+    public CommitMitigatingTieredMergePolicy setSegmentsPerTier(double v) {
+        if (v < 2.0) {
+            throw new IllegalArgumentException("segmentsPerTier must be >= 2.0 (got " + v + ")");
+        }
+        segsPerTier = v;
+        return this;
+    }
+
+    /** Returns the current segmentsPerTier setting.
+     *
+     * @see #setSegmentsPerTier */
+    public double getSegmentsPerTier() {
+        return segsPerTier;
+    }
+
+    public CommitMitigatingTieredMergePolicy setMaxCommitRateMB(int maxCommitRateMB) {
+        this.maxCommitRateMB = maxCommitRateMB;
+        return this;
+    }
+
+    private class SegmentByteSizeDescending implements Comparator<SegmentCommitInfo> {
+        @Override
+        public int compare(SegmentCommitInfo o1, SegmentCommitInfo o2) {
+            try {
+                final long sz1 = size(o1);
+                final long sz2 = size(o2);
+                if (sz1 > sz2) {
+                    return -1;
+                } else if (sz2 > sz1) {
+                    return 1;
+                } else {
+                    return o1.info.name.compareTo(o2.info.name);
+                }
+            } catch (IOException ioe) {
+                throw new RuntimeException(ioe);
+            }
+        }
+    }
+
+    /** Holds score and explanation for a single candidate
+     *  merge. */
+    protected static abstract class MergeScore {
+        /** Sole constructor. (For invocation by subclass
+         *  constructors, typically implicit.) */
+        protected MergeScore() {
+        }
+
+        /** Returns the score for this merge candidate; lower
+         *  scores are better. */
+        abstract double getScore();
+
+        /** Human readable explanation of how the merge got this
+         *  score. */
+        abstract String getExplanation();
+    }
+
+    @Override
+    public MergeSpecification findMerges(MergeTrigger mergeTrigger, SegmentInfos infos) throws IOException {
+        if (verbose()) {
+            message("findMerges: " + infos.size() + " segments");
+        }
+        if (infos.size() == 0) {
+            return null;
+        }
+
+        long now = System.currentTimeMillis();
+        double timeDelta = (now / 1000d) - (time / 1000d);
+        double commitRate = Math.abs(docCount - infos.totalDocCount()) / timeDelta;
+
+        docCount = infos.totalDocCount();
+        time = now;
+
+        if (verbose()) {
+            message(commitRate + "doc/s (max: " + maxCommitRateDocs + "doc/s)");
+        }
+
+        if (commitRate > maxCommitRateDocs) {
+            return null;
+        }
+
+        final Collection<SegmentCommitInfo> merging = writer.get().getMergingSegments();
+        final Collection<SegmentCommitInfo> toBeMerged = new HashSet<SegmentCommitInfo>();
+
+        final List<SegmentCommitInfo> infosSorted = new ArrayList<SegmentCommitInfo>(infos.asList());
+        Collections.sort(infosSorted, new SegmentByteSizeDescending());
+
+        // Compute total index bytes & print details about the index
+        long totIndexBytes = 0;
+        long minSegmentBytes = Long.MAX_VALUE;
+        for (SegmentCommitInfo info : infosSorted) {
+            final long segBytes = size(info);
+            if (verbose()) {
+                String extra = merging.contains(info) ? " [merging]" : "";
+                if (segBytes >= maxMergedSegmentBytes / 2.0) {
+                    extra += " [skip: too large]";
+                } else if (segBytes < floorSegmentBytes) {
+                    extra += " [floored]";
+                }
+                message("  seg=" + writer.get().segString(info) + " size=" + String.format(Locale.ROOT, "%.3f", segBytes / 1024 / 1024.) + " MB" + extra);
+            }
+
+            minSegmentBytes = Math.min(segBytes, minSegmentBytes);
+            // Accum total byte size
+            totIndexBytes += segBytes;
+        }
+
+        // If we have too-large segments, grace them out
+        // of the maxSegmentCount:
+        int tooBigCount = 0;
+        while (tooBigCount < infosSorted.size() && size(infosSorted.get(tooBigCount)) >= maxMergedSegmentBytes / 2.0) {
+            totIndexBytes -= size(infosSorted.get(tooBigCount));
+            tooBigCount++;
+        }
+
+        minSegmentBytes = floorSize(minSegmentBytes);
+
+        // Compute max allowed segs in the index
+        long levelSize = minSegmentBytes;
+        long bytesLeft = totIndexBytes;
+        double allowedSegCount = 0;
+        while (true) {
+            final double segCountLevel = bytesLeft / (double) levelSize;
+            if (segCountLevel < segsPerTier) {
+                allowedSegCount += Math.ceil(segCountLevel);
+                break;
+            }
+            allowedSegCount += segsPerTier;
+            bytesLeft -= segsPerTier * levelSize;
+            levelSize *= maxMergeAtOnce;
+        }
+        int allowedSegCountInt = (int) allowedSegCount;
+
+        MergeSpecification spec = null;
+
+        // Cycle to possibly select more than one merge:
+        while (true) {
+
+            long mergingBytes = 0;
+            double idxBytes = 0;
+
+            // Gather eligible segments for merging, ie segments
+            // not already being merged and not already picked (by
+            // prior iteration of this loop) for merging:
+            final List<SegmentCommitInfo> eligible = new ArrayList<SegmentCommitInfo>();
+            for (int idx = tooBigCount; idx < infosSorted.size(); idx++) {
+                final SegmentCommitInfo info = infosSorted.get(idx);
+                if (merging.contains(info)) {
+                    mergingBytes += info.sizeInBytes();
+                } else if (!toBeMerged.contains(info)) {
+                    eligible.add(info);
+                }
+                idxBytes += info.sizeInBytes();
+            }
+            idxBytes /= 1024 * 1000;
+
+            final boolean maxMergeIsRunning = mergingBytes >= maxMergedSegmentBytes;
+
+            if (verbose()) {
+                message("  allowedSegmentCount=" + allowedSegCountInt + " vs count=" + infosSorted.size() + " (eligible count=" + eligible.size() + ") tooBigCount=" + tooBigCount);
+            }
+
+            if (eligible.size() == 0) {
+                return spec;
+            }
+
+            double bytes = idxBytes - this.mb;
+            double mbRate = bytes / timeDelta;
+
+            if (verbose()) {
+                message(mbRate + "mb/s (max: " + maxCommitRateMB + "mb/s)");
+            }
+
+            this.mb = idxBytes;
+            if (mbRate > maxCommitRateMB) {
+                return null;
+            }
+
+            if (eligible.size() >= allowedSegCountInt) {
+
+                // OK we are over budget -- find best merge!
+                MergeScore bestScore = null;
+                List<SegmentCommitInfo> best = null;
+                boolean bestTooLarge = false;
+                long bestMergeBytes = 0;
+
+                // Consider all merge starts:
+                for (int startIdx = 0; startIdx <= eligible.size() - maxMergeAtOnce; startIdx++) {
+
+                    long totAfterMergeBytes = 0;
+
+                    final List<SegmentCommitInfo> candidate = new ArrayList<SegmentCommitInfo>();
+                    boolean hitTooLarge = false;
+                    for (int idx = startIdx; idx < eligible.size() && candidate.size() < maxMergeAtOnce; idx++) {
+                        final SegmentCommitInfo info = eligible.get(idx);
+                        final long segBytes = size(info);
+
+                        if (totAfterMergeBytes + segBytes > maxMergedSegmentBytes) {
+                            hitTooLarge = true;
+                            // NOTE: we continue, so that we can try
+                            // "packing" smaller segments into this merge
+                            // to see if we can get closer to the max
+                            // size; this in general is not perfect since
+                            // this is really "bin packing" and we'd have
+                            // to try different permutations.
+                            continue;
+                        }
+                        candidate.add(info);
+                        totAfterMergeBytes += segBytes;
+                    }
+
+                    final MergeScore score = score(candidate, hitTooLarge, mergingBytes);
+                    if (verbose()) {
+                        message("  maybe=" + writer.get().segString(candidate) + " score=" + score.getScore() + " " + score.getExplanation() + " tooLarge=" + hitTooLarge + " size=" + String.format(Locale.ROOT, "%.3f MB", totAfterMergeBytes / 1024. / 1024.));
+                    }
+
+                    // If we are already running a max sized merge
+                    // (maxMergeIsRunning), don't allow another max
+                    // sized merge to kick off:
+                    if ((bestScore == null || score.getScore() < bestScore.getScore()) && (!hitTooLarge || !maxMergeIsRunning)) {
+                        best = candidate;
+                        bestScore = score;
+                        bestTooLarge = hitTooLarge;
+                        bestMergeBytes = totAfterMergeBytes;
+                    }
+                }
+
+                if (best != null) {
+                    if (spec == null) {
+                        spec = new MergeSpecification();
+                    }
+                    final OneMerge merge = new OneMerge(best);
+                    spec.add(merge);
+                    for (SegmentCommitInfo info : merge.segments) {
+                        toBeMerged.add(info);
+                    }
+                    if (verbose()) {
+                        message("  add merge=" + writer.get().segString(merge.segments) + " size=" + String.format(Locale.ROOT, "%.3f MB", bestMergeBytes / 1024. / 1024.) + " score=" + String.format(Locale.ROOT, "%.3f", bestScore.getScore()) + " " + bestScore.getExplanation() + (bestTooLarge ? " [max merge]" : ""));
+                    }
+                } else {
+                    return spec;
+                }
+            } else {
+                return spec;
+            }
+        }
+    }
+
+    /** Expert: scores one merge; subclasses can override. */
+    protected MergeScore score(List<SegmentCommitInfo> candidate, boolean hitTooLarge, long mergingBytes) throws IOException {
+        long totBeforeMergeBytes = 0;
+        long totAfterMergeBytes = 0;
+        long totAfterMergeBytesFloored = 0;
+        for (SegmentCommitInfo info : candidate) {
+            final long segBytes = size(info);
+            totAfterMergeBytes += segBytes;
+            totAfterMergeBytesFloored += floorSize(segBytes);
+            totBeforeMergeBytes += info.sizeInBytes();
+        }
+
+        // Roughly measure "skew" of the merge, i.e. how
+        // "balanced" the merge is (whether the segments are
+        // about the same size), which can range from
+        // 1.0/numSegsBeingMerged (good) to 1.0 (poor). Heavily
+        // lopsided merges (skew near 1.0) is no good; it means
+        // O(N^2) merge cost over time:
+        final double skew;
+        if (hitTooLarge) {
+            // Pretend the merge has perfect skew; skew doesn't
+            // matter in this case because this merge will not
+            // "cascade" and so it cannot lead to N^2 merge cost
+            // over time:
+            skew = 1.0 / maxMergeAtOnce;
+        } else {
+            skew = ((double) floorSize(size(candidate.get(0)))) / totAfterMergeBytesFloored;
+        }
+
+        // Strongly favor merges with less skew (smaller
+        // mergeScore is better):
+        double mergeScore = skew;
+
+        // Gently favor smaller merges over bigger ones.  We
+        // don't want to make this exponent too large else we
+        // can end up doing poor merges of small segments in
+        // order to avoid the large merges:
+        mergeScore *= Math.pow(totAfterMergeBytes, 0.05);
+
+        // Strongly favor merges that reclaim deletes:
+        final double nonDelRatio = ((double) totAfterMergeBytes) / totBeforeMergeBytes;
+        mergeScore *= Math.pow(nonDelRatio, reclaimDeletesWeight);
+
+        final double finalMergeScore = mergeScore;
+
+        return new MergeScore() {
+
+            @Override
+            public double getScore() {
+                return finalMergeScore;
+            }
+
+            @Override
+            public String getExplanation() {
+                return "skew=" + String.format(Locale.ROOT, "%.3f", skew) + " nonDelRatio=" + String.format(Locale.ROOT, "%.3f", nonDelRatio);
+            }
+        };
+    }
+
+    @Override
+    public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCount, Map<SegmentCommitInfo, Boolean> segmentsToMerge) throws IOException {
+        if (verbose()) {
+            message("findForcedMerges maxSegmentCount=" + maxSegmentCount + " infos=" + writer.get().segString(infos) + " segmentsToMerge=" + segmentsToMerge);
+        }
+
+        List<SegmentCommitInfo> eligible = new ArrayList<SegmentCommitInfo>();
+        boolean forceMergeRunning = false;
+        final Collection<SegmentCommitInfo> merging = writer.get().getMergingSegments();
+        boolean segmentIsOriginal = false;
+        for (SegmentCommitInfo info : infos) {
+            final Boolean isOriginal = segmentsToMerge.get(info);
+            if (isOriginal != null) {
+                segmentIsOriginal = isOriginal;
+                if (!merging.contains(info)) {
+                    eligible.add(info);
+                } else {
+                    forceMergeRunning = true;
+                }
+            }
+        }
+
+        if (eligible.size() == 0) {
+            return null;
+        }
+
+        if ((maxSegmentCount > 1 && eligible.size() <= maxSegmentCount) ||
+                (maxSegmentCount == 1 && eligible.size() == 1 && (!segmentIsOriginal || isMerged(infos, eligible.get(0))))) {
+            if (verbose()) {
+                message("already merged");
+            }
+            return null;
+        }
+
+        Collections.sort(eligible, new SegmentByteSizeDescending());
+
+        if (verbose()) {
+            message("eligible=" + eligible);
+            message("forceMergeRunning=" + forceMergeRunning);
+        }
+
+        int end = eligible.size();
+
+        MergeSpecification spec = null;
+
+        // Do full merges, first, backwards:
+        while (end >= maxMergeAtOnceExplicit + maxSegmentCount - 1) {
+            if (spec == null) {
+                spec = new MergeSpecification();
+            }
+            final OneMerge merge = new OneMerge(eligible.subList(end - maxMergeAtOnceExplicit, end));
+            if (verbose()) {
+                message("add merge=" + writer.get().segString(merge.segments));
+            }
+            spec.add(merge);
+            end -= maxMergeAtOnceExplicit;
+        }
+
+        if (spec == null && !forceMergeRunning) {
+            // Do final merge
+            final int numToMerge = end - maxSegmentCount + 1;
+            final OneMerge merge = new OneMerge(eligible.subList(end - numToMerge, end));
+            if (verbose()) {
+                message("add final merge=" + merge.segString(writer.get().getDirectory()));
+            }
+            spec = new MergeSpecification();
+            spec.add(merge);
+        }
+
+        return spec;
+    }
+
+    @Override
+    public MergeSpecification findForcedDeletesMerges(SegmentInfos infos) throws IOException {
+        if (verbose()) {
+            message("findForcedDeletesMerges infos=" + writer.get().segString(infos) + " forceMergeDeletesPctAllowed=" + forceMergeDeletesPctAllowed);
+        }
+        final List<SegmentCommitInfo> eligible = new ArrayList<SegmentCommitInfo>();
+        final Collection<SegmentCommitInfo> merging = writer.get().getMergingSegments();
+        for (SegmentCommitInfo info : infos) {
+            double pctDeletes = 100. * ((double) writer.get().numDeletedDocs(info)) / info.info.getDocCount();
+            if (pctDeletes > forceMergeDeletesPctAllowed && !merging.contains(info)) {
+                eligible.add(info);
+            }
+        }
+
+        if (eligible.size() == 0) {
+            return null;
+        }
+
+        Collections.sort(eligible, new SegmentByteSizeDescending());
+
+        if (verbose()) {
+            message("eligible=" + eligible);
+        }
+
+        int start = 0;
+        MergeSpecification spec = null;
+
+        while (start < eligible.size()) {
+            // Don't enforce max merged size here: app is explicitly
+            // calling forceMergeDeletes, and knows this may take a
+            // long time / produce big segments (like forceMerge):
+            final int end = Math.min(start + maxMergeAtOnceExplicit, eligible.size());
+            if (spec == null) {
+                spec = new MergeSpecification();
+            }
+
+            final OneMerge merge = new OneMerge(eligible.subList(start, end));
+            if (verbose()) {
+                message("add merge=" + writer.get().segString(merge.segments));
+            }
+            spec.add(merge);
+            start = end;
+        }
+
+        return spec;
+    }
+
+    @Override
+    public void close() {
+    }
+
+    private long floorSize(long bytes) {
+        return Math.max(floorSegmentBytes, bytes);
+    }
+
+    private boolean verbose() {
+        final IndexWriter w = writer.get();
+        return w != null && w.getConfig().getInfoStream().isEnabled("TMP");
+    }
+
+    private void message(String message) {
+        writer.get().getConfig().getInfoStream().message("TMP", message);
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder("[" + getClass().getSimpleName() + ": ");
+        sb.append("maxMergeAtOnce=").append(maxMergeAtOnce).append(", ");
+        sb.append("maxMergeAtOnceExplicit=").append(maxMergeAtOnceExplicit).append(", ");
+        sb.append("maxMergedSegmentMB=").append(maxMergedSegmentBytes / 1024 / 1024.).append(", ");
+        sb.append("floorSegmentMB=").append(floorSegmentBytes / 1024 / 1024.).append(", ");
+        sb.append("forceMergeDeletesPctAllowed=").append(forceMergeDeletesPctAllowed).append(", ");
+        sb.append("segmentsPerTier=").append(segsPerTier).append(", ");
+        sb.append("maxCFSSegmentSizeMB=").append(getMaxCFSSegmentSizeMB()).append(", ");
+        sb.append("noCFSRatio=").append(noCFSRatio);
+        return sb.toString();
+    }
+}

Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/CommitMitigatingTieredMergePolicy.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java?rev=1806013&r1=1806012&r2=1806013&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java Thu Aug 24 09:15:10 2017
@@ -148,6 +148,7 @@ class DefaultIndexWriter implements Luce
             } else {
                 config = getIndexWriterConfig(definition, true);
             }
+            config.setMergePolicy(definition.getMergePolicy());
             writer = new IndexWriter(directory, config);
             genAtStart = getLatestGeneration(directory);
             PERF_LOGGER.end(start, -1, "Created IndexWriter for directory {}", definition);
@@ -159,7 +160,6 @@ class DefaultIndexWriter implements Luce
      * eventually update suggest dictionary
      * @throws IOException if suggest dictionary update fails
      * @param analyzer the analyzer used to update the suggester
-     * @param blobStore
      */
     private boolean updateSuggester(Analyzer analyzer, Calendar currentTime) throws IOException {
         NodeBuilder suggesterStatus = definitionBuilder.child(suggestDirName);

Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java?rev=1806013&r1=1806012&r2=1806013&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java Thu Aug 24 09:15:10 2017
@@ -51,6 +51,7 @@ public class IndexWriterUtils {
             }
             Analyzer analyzer = new PerFieldAnalyzerWrapper(definitionAnalyzer, analyzers);
             IndexWriterConfig config = new IndexWriterConfig(VERSION, analyzer);
+//            config.setMergePolicy(definition.getMergePolicy());
             if (remoteDir) {
                 config.setMergeScheduler(new SerialMergeScheduler());
             }

Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java?rev=1806013&r1=1806012&r2=1806013&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java Thu Aug 24 09:15:10 2017
@@ -30,9 +30,14 @@ import org.apache.jackrabbit.oak.plugins
 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule;
 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.IndexingMode;
 import org.apache.jackrabbit.oak.plugins.index.lucene.util.TokenizerChain;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.CommitMitigatingTieredMergePolicy;
 import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
 import org.apache.jackrabbit.oak.spi.state.NodeState;
 import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.index.LogByteSizeMergePolicy;
+import org.apache.lucene.index.LogDocMergePolicy;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
 import org.junit.Test;
 
 import static com.google.common.collect.ImmutableSet.of;
@@ -154,6 +159,49 @@ public class IndexDefinitionTest {
     }
 
     @Test
+    public void mergePolicyConfig() throws Exception{
+        IndexDefinition defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+        assertNotNull(defn.getMergePolicy());
+        assertEquals(CommitMitigatingTieredMergePolicy.class, defn.getMergePolicy().getClass());
+
+        builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "tiered");
+        defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+        assertNotNull(defn.getMergePolicy());
+        assertEquals(TieredMergePolicy.class, defn.getMergePolicy().getClass());
+
+        builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "logbyte");
+        defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+        assertNotNull(defn.getMergePolicy());
+        assertEquals(LogByteSizeMergePolicy.class, defn.getMergePolicy().getClass());
+
+        builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "logdoc");
+        defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+        assertNotNull(defn.getMergePolicy());
+        assertEquals(LogDocMergePolicy.class, defn.getMergePolicy().getClass());
+
+        builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "no");
+        defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+        assertNotNull(defn.getMergePolicy());
+        assertEquals(NoMergePolicy.class, defn.getMergePolicy().getClass());
+
+        builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "default");
+        defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+        assertNotNull(defn.getMergePolicy());
+        assertEquals(CommitMitigatingTieredMergePolicy.class, defn.getMergePolicy().getClass());
+
+        builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "mitigated");
+        defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+        assertNotNull(defn.getMergePolicy());
+        assertEquals(CommitMitigatingTieredMergePolicy.class, defn.getMergePolicy().getClass());
+
+        // wrong mp name falls back to default
+        builder.setProperty(LuceneIndexConstants.MERGE_POLICY_NAME, "whoawhoa");
+        defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
+        assertNotNull(defn.getMergePolicy());
+        assertEquals(CommitMitigatingTieredMergePolicy.class, defn.getMergePolicy().getClass());
+    }
+
+    @Test
     public void relativePropertyConfig() throws Exception{
         builder.child(PROP_NODE).child("foo1").child("bar").setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_DATE);
         builder.child(PROP_NODE).child("foo2").child("bar2").child("baz").setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_LONG);

Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java?rev=1806013&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java Thu Aug 24 09:15:10 2017
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+
+import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.core.data.FileDataStore;
+import org.apache.jackrabbit.oak.InitialContent;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.api.ContentRepository;
+import org.apache.jackrabbit.oak.api.Tree;
+import org.apache.jackrabbit.oak.commons.IOUtils;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.plugins.blob.BlobStoreStats;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.CopyOnReadDirectory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+import org.apache.jackrabbit.oak.query.AbstractQueryTest;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
+import org.apache.jackrabbit.oak.segment.file.FileStoreStats;
+import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.blob.stats.BlobStatsCollector;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.FilterDirectory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/**
+ * Tests for checking impacts of Lucene writes wrt storage / configuration adjustments on the
+ * {@link org.apache.jackrabbit.oak.segment.SegmentNodeStore}.
+ */
+@Ignore("this is meant to be a benchmark, it shouldn't be part of everyday builds")
+@RunWith(Parameterized.class)
+public class LuceneWritesOnSegmentStatsTest extends AbstractQueryTest {
+
+    private static final File DIRECTORY = new File("target/fs");
+
+    private final boolean copyOnRW;
+    private final String codec;
+    private final boolean indexOnFS;
+    private final int minRecordLength;
+    private final String mergePolicy;
+
+    private ExecutorService executorService = Executors.newFixedThreadPool(2);
+
+    private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
+
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target"));
+
+    private String corDir = null;
+    private String cowDir = null;
+
+    private TestUtil.OptionalEditorProvider optionalEditorProvider = new TestUtil.OptionalEditorProvider();
+
+    private FileStore fileStore;
+    private DataStoreBlobStore dataStoreBlobStore;
+    private DefaultStatisticsProvider statisticsProvider;
+    private String fdsDir;
+    private String indexPath;
+
+
+    public LuceneWritesOnSegmentStatsTest(boolean copyOnRW, String codec, boolean indexOnFS, int minRecordLength, String mergePolicy) {
+        this.copyOnRW = copyOnRW;
+        this.codec = codec;
+        this.indexOnFS = indexOnFS;
+        this.minRecordLength = minRecordLength;
+        this.mergePolicy = mergePolicy;
+    }
+
+    @Parameterized.Parameters
+    public static Collection<Object[]> data() {
+        return Arrays.asList(new Object[][]{
+                {false, "oakCodec", false, 4000, "tiered"},
+                {false, "oakCodec", false, 4000, "mitigated"},
+                {false, "oakCodec", false, 4000, "no"},
+                {false, "Lucene46", false, 4000, "tiered"},
+                {false, "Lucene46", false, 4000, "mitigated"},
+                {false, "Lucene46", false, 4000, "no"},
+                {false, "oakCodec", false, -1, "tiered"},
+                {false, "oakCodec", false, -1, "mitigated"},
+                {false, "oakCodec", false, -1, "no"},
+                {false, "Lucene46", false, -1, "tiered"},
+                {false, "Lucene46", false, -1, "mitigated"},
+                {false, "Lucene46", false, -1, "no"},
+        });
+    }
+
+    @Before
+    public void setUp() throws Exception {
+        if (!DIRECTORY.exists()) {
+            assert DIRECTORY.mkdirs();
+        }
+    }
+
+    @After
+    public void after() {
+        new ExecutorCloser(executorService).close();
+        IndexDefinition.setDisableStoredIndexDefinition(false);
+        fileStore.close();
+        if (DIRECTORY.exists()) {
+            try {
+                FileUtils.deleteDirectory(DIRECTORY);
+            } catch (IOException e) {
+                // do nothing
+            }
+        }
+    }
+
+    @Override
+    protected void createTestIndexNode() throws Exception {
+        setTraversalEnabled(false);
+    }
+
+    @Override
+    protected ContentRepository createRepository() {
+        LuceneIndexEditorProvider editorProvider;
+        LuceneIndexProvider provider;
+        if (copyOnRW) {
+            IndexCopier copier = createIndexCopier();
+            editorProvider = new LuceneIndexEditorProvider(copier, new ExtractedTextCache(10 * FileUtils.ONE_MB, 100));
+            provider = new LuceneIndexProvider(copier);
+        } else {
+            editorProvider = new LuceneIndexEditorProvider();
+            provider = new LuceneIndexProvider();
+        }
+
+        NodeStore nodeStore;
+        try {
+            statisticsProvider = new DefaultStatisticsProvider(scheduledExecutorService);
+            fileStore = FileStoreBuilder.fileStoreBuilder(DIRECTORY)
+                    .withStatisticsProvider(statisticsProvider)
+                    .withBlobStore(createBlobStore())
+                    .build();
+            nodeStore = SegmentNodeStoreBuilders.builder(fileStore).build();
+        } catch (IOException | InvalidFileStoreVersionException e) {
+            throw new RuntimeException(e);
+        }
+        return new Oak(nodeStore)
+                .with(new InitialContent())
+                .with(new OpenSecurityProvider())
+                .with((QueryIndexProvider) provider)
+                .with((Observer) provider)
+                .with(editorProvider)
+                .with(optionalEditorProvider)
+                .with(new PropertyIndexEditorProvider())
+                .with(new NodeTypeIndexProvider())
+                .createContentRepository();
+    }
+
+    private BlobStore createBlobStore() {
+        FileDataStore fds = new OakFileDataStore();
+        fdsDir = "target/fds-" + codec + copyOnRW + minRecordLength + mergePolicy;
+        fds.setPath(fdsDir);
+        if (minRecordLength > 0) {
+            fds.setMinRecordLength(minRecordLength);
+        }
+        fds.init(null);
+        dataStoreBlobStore = new DataStoreBlobStore(fds);
+        StatisticsProvider sp = new DefaultStatisticsProvider(scheduledExecutorService);
+        BlobStatsCollector collector = new BlobStoreStats(sp);
+        dataStoreBlobStore.setBlobStatsCollector(collector);
+        return dataStoreBlobStore;
+    }
+
+    private IndexCopier createIndexCopier() {
+        try {
+            return new IndexCopier(executorService, temporaryFolder.getRoot()) {
+                @Override
+                public Directory wrapForRead(String indexPath, IndexDefinition definition,
+                                             Directory remote, String dirName) throws IOException {
+                    Directory ret = super.wrapForRead(indexPath, definition, remote, dirName);
+                    corDir = getFSDirPath(ret);
+                    return ret;
+                }
+
+                @Override
+                public Directory wrapForWrite(IndexDefinition definition,
+                                              Directory remote, boolean reindexMode, String dirName) throws IOException {
+                    Directory ret = super.wrapForWrite(definition, remote, reindexMode, dirName);
+                    cowDir = getFSDirPath(ret);
+                    return ret;
+                }
+
+                private String getFSDirPath(Directory dir) {
+                    if (dir instanceof CopyOnReadDirectory) {
+                        dir = ((CopyOnReadDirectory) dir).getLocal();
+                    }
+
+                    dir = unwrap(dir);
+
+                    if (dir instanceof FSDirectory) {
+                        return ((FSDirectory) dir).getDirectory().getAbsolutePath();
+                    }
+                    return null;
+                }
+
+                private Directory unwrap(Directory dir) {
+                    if (dir instanceof FilterDirectory) {
+                        return unwrap(((FilterDirectory) dir).getDelegate());
+                    }
+                    return dir;
+                }
+
+            };
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @After
+    public void shutdownExecutor() {
+        executorService.shutdown();
+        scheduledExecutorService.shutdown();
+    }
+
+    @Test
+    public void testLuceneIndexSegmentStats() throws Exception {
+        IndexDefinitionBuilder idxb = new IndexDefinitionBuilder().noAsync().codec(codec).mergePolicy(mergePolicy);
+        idxb.indexRule("nt:base").property("foo").analyzed().nodeScopeIndex().ordered().useInExcerpt().propertyIndex();
+        idxb.indexRule("nt:base").property("bin").analyzed().nodeScopeIndex().ordered().useInExcerpt().propertyIndex();
+        Tree idx = root.getTree("/").getChild("oak:index").addChild("lucenePropertyIndex");
+        Tree idxDef = idxb.build(idx);
+        if (!codec.equals("oakCodec") && indexOnFS) {
+            idxDef.setProperty("persistence", "file");
+            indexPath = "target/index-" + codec + copyOnRW;
+            idxDef.setProperty("path", indexPath);
+        }
+
+        Random r = new Random();
+
+        System.out.println("***");
+        System.out.println(codec + "," + copyOnRW + "," + indexOnFS + "," + minRecordLength + "," + mergePolicy);
+
+        long start = System.currentTimeMillis();
+        int multiplier = 3;
+        for (int n = 0; n < multiplier; n++) {
+            System.err.println("iteration " + (n + 1));
+
+            Tree rootTree = root.getTree("/").addChild("content");
+            byte[] bytes = new byte[10240];
+            Charset charset = Charset.defaultCharset();
+            for (int i = 0; i < 1000; i++) {
+                r.nextBytes(bytes);
+                String text = new String(bytes, charset);
+                Tree tree = rootTree.addChild(String.valueOf(n + i));
+                tree.setProperty("foo", text);
+                tree.setProperty("bin", bytes);
+            }
+            root.commit();
+
+            printStats();
+
+            System.out.println("reindex");
+
+            // do nothing, reindex and measure
+            idx = root.getTree("/oak:index/lucenePropertyIndex");
+            idx.setProperty("reindex", true);
+            root.commit();
+
+            printStats();
+
+            System.out.println("add and delete");
+
+            // add and delete some content and measure
+            for (int i = 0; i < 1000; i++) {
+                r.nextBytes(bytes);
+                String text = new String(bytes, charset);
+                Tree tree = rootTree.addChild(String.valueOf(n + 100 + i));
+                tree.setProperty("foo", text);
+                tree.setProperty("bin", bytes);
+                if (n + i % 3 == 0) { // delete one of the already existing nodes every 3
+                    assert rootTree.getChild(String.valueOf(n + i)).remove();
+                }
+            }
+            root.commit();
+
+            printStats();
+        }
+        long time = System.currentTimeMillis() - start;
+        System.out.println("finished in " + (time / (60000)) + " minutes");
+        System.out.println("***");
+    }
+
+    private void printStats() throws IOException {
+        fileStore.flush();
+
+        FileStoreStats stats = fileStore.getStats();
+
+        long sizeOfDirectory = FileUtils.sizeOfDirectory(new File(fdsDir));
+        String fdsSize = (sizeOfDirectory / (1024 * 1000)) + " MB";
+
+        System.err.println("||codec||min record length||merge policy||segment size||FDS size||");
+        System.err.println("|"+codec+"|"+minRecordLength+"|"+mergePolicy+"|"+ IOUtils.humanReadableByteCount(stats.getApproximateSize())+"|"+fdsSize+"|");
+
+        if (indexOnFS) {
+            long sizeOfFSIndex = FileUtils.sizeOfDirectory(new File(indexPath));
+            System.out.println("Index on FS size : " + FileUtils.byteCountToDisplaySize(sizeOfFSIndex));
+        }
+    }
+
+    private long dumpFileStoreTo(File to) throws IOException {
+        if (!to.exists()) {
+            assert to.mkdirs();
+        }
+        for (File f : DIRECTORY.listFiles()) {
+            Files.copy(f, new File(to.getPath(), f.getName()));
+        }
+
+        long sizeOfDirectory = FileUtils.sizeOfDirectory(to);
+
+        to.deleteOnExit();
+        return sizeOfDirectory;
+    }
+}

Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneWritesOnSegmentStatsTest.java
------------------------------------------------------------------------------
    svn:eol-style = native