You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/11/04 18:49:19 UTC
svn commit: r1031101 - in
/lucene/dev/branches/branch_3x/lucene/contrib/benchmark: ./ conf/
src/java/org/apache/lucene/benchmark/byTask/tasks/
src/test/org/apache/lucene/benchmark/byTask/tasks/
Author: simonw
Date: Thu Nov 4 17:49:19 2010
New Revision: 1031101
URL: http://svn.apache.org/viewvc?rev=1031101&view=rev
Log:
LUCENE-2734: Use IndexWriterConfig in benchmark
Modified:
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-flush-by-RAM-multithreaded.alg
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-flush-by-RAM.alg
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-multithreaded.alg
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing.alg
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/standard.alg
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTaskTest.java
Propchange: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Thu Nov 4 17:49:19 2010
@@ -0,0 +1,8 @@
+/lucene/dev/branches/preflexfixes/lucene/contrib/benchmark:967125-979432
+/lucene/dev/trunk/lucene/contrib/benchmark:931298,931337,931502,932129-932131,932163,932304,932369,932374,932398,932417,932541,932576,932587,932698,932731-932749,932752,932773,932795,932828,932856-932857,932862,932864,932878,932963,932998-932999,933541-933575,933598,933613,933679,933879,934339,934954,935014-935048,935065,935186-935513,935521-935522,935553-935962,936522,936544,936605,936657-936726,937039,937360,938582-938646,938989,939111,939611,939649,940433,940447,940451-940452,940666,940699,940730,940878-940892,940994,941270,941363,942166,942235,942288,942676,942719,943142,943493,943931,945057,945090,945130,945245,945343,945420,946139,946330,946338,946599,948011,948082,948429,949288,949311,949445,949976,949997,950008,950042,950458,950613,951126,951355,951397,951521,953628,955547,955613,955615,955796-955797,955809-955996,956097,956125,956173,956316,956715,957465,957520,957634,957707,960367,960371,960374,960719,962555,963372,963654,963720,963781,963873,963906,963909,963920,9
64019,964054,964430,964459,964720,964753,964832,964856,965103,965110,965222,965230,965299,965327,965330,965585,966354,966878,967080,979453,979809,980369,980428,980436,980501,980909,980911,980917,981265,981550,981598,981650,981661,981857,981936,982073,982084,982201,982725,982824,983100,983212,983216,983313,983495,983500,983530,983622,983632,983778,984187,984202,984232,984510,984968,985453,985455,985672,985875,986158,986173,986612,987122,988087,988206,988216,988259,988346,988478,988527,988543,988592,988613,988688,988710,988736,988739,989004,989010,989013,989030,989035,989315,989321,989334,989785,990160-990161,990180,990189,990281,990301,990451,990459,990766,990781,990854,991053,991191,991310,991497,992424,992469,992567,992571,992623,993106,993194,993199,993287,993408,994935,994976,994979,995247,995250,995376,995772,996268,996357,996416,996511,996611,996623,996647-996653,996720,996942,996978,997180,997230,998055,998505,998684,999016,999223,999545,999842,1000424,1000581,1000675,
1001006,1001420,1001661,1001796,1002032,1003614,1003631,1003645,1003841-1003852,1003873,1003877,1003906,1003938,1003954,1003978,1003990,1004038,1004082,1004179,1004200,1004215,1004241,1004335,1005310,1005356,1005363,1006146,1006290,1006324,1021340,1021357,1021360,1021439,1021449,1021969-1021971,1022165,1022191,1022632,1022708-1022710,1022730-1022735,1022748-1022755,1022762-1022793,1022798-1022802,1022805,1022826,1022927,1022939,1022956,1022989,1022998,1023006,1023009,1023022,1023040,1023106,1023235-1023246,1023250,1023264-1023265,1023312,1023329-1023330,1023346-1023347,1023355,1023493,1023509-1023511,1023518,1023520,1023535-1023536,1023562,1023579-1023588,1023594-1023595,1023600-1023602,1023606,1023621,1023635,1023637,1023711,1023845,1023870,1024196,1024219,1024233,1024238,1024256,1024292,1024305,1024395,1024402,1024408,1024476,1024486,1025597,1025669,1025929,1026044,1026167,1026336,1026431,1026460,1026592,1026606,1026610,1026738,1026841,1026868,1026882,1027998,1028039,10283
86,1029096,1029325,1029333,1029345,1030012,1030019,1030073,1030078,1030754,1031076,1031097
+/lucene/java/branches/flex_1458/contrib/benchmark:924791,924850,930201
+/lucene/java/branches/lucene_2_4/contrib/benchmark:748824
+/lucene/java/branches/lucene_2_9/contrib/benchmark:817269-818600,825998,829134,829816,829881,831036,896850,909334,948516
+/lucene/java/branches/lucene_2_9_back_compat_tests/contrib/benchmark:818601-821336
+/lucene/java/branches/lucene_3_0/contrib/benchmark:880793,896906
+/lucene/java/trunk/contrib/benchmark:924483-925561
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-flush-by-RAM-multithreaded.alg
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-flush-by-RAM-multithreaded.alg?rev=1031101&r1=1031100&r2=1031101&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-flush-by-RAM-multithreaded.alg (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-flush-by-RAM-multithreaded.alg Thu Nov 4 17:49:19 2010
@@ -17,6 +17,7 @@
# -------------------------------------------------------------------------------------
# multi val params are iterated by NewRound's, added to reports, start with column name.
+writer.version=LUCENE_31
#merge.factor=mrg:10:100:10:100:10:100:10:100
#max.buffered=buf:10:10:100:100:10:10:100:100
ram.flush.mb=flush:32:40:48:56:32:40:48:56
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-flush-by-RAM.alg
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-flush-by-RAM.alg?rev=1031101&r1=1031100&r2=1031101&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-flush-by-RAM.alg (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-flush-by-RAM.alg Thu Nov 4 17:49:19 2010
@@ -17,6 +17,7 @@
# -------------------------------------------------------------------------------------
# multi val params are iterated by NewRound's, added to reports, start with column name.
+writer.version=LUCENE_31
#merge.factor=mrg:10:100:10:100:10:100:10:100
#max.buffered=buf:10:10:100:100:10:10:100:100
ram.flush.mb=flush:32:40:48:56:32:40:48:56
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-multithreaded.alg
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-multithreaded.alg?rev=1031101&r1=1031100&r2=1031101&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-multithreaded.alg (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing-multithreaded.alg Thu Nov 4 17:49:19 2010
@@ -17,6 +17,7 @@
# -------------------------------------------------------------------------------------
# multi val params are iterated by NewRound's, added to reports, start with column name.
+writer.version=LUCENE_31
merge.factor=mrg:10:100:10:100:10:100:10:100
max.buffered=buf:10:10:100:100:10:10:100:100
#ram.flush.mb=flush:32:40:48:56:32:40:48:56
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing.alg
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing.alg?rev=1031101&r1=1031100&r2=1031101&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing.alg (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/indexing.alg Thu Nov 4 17:49:19 2010
@@ -17,6 +17,7 @@
# -------------------------------------------------------------------------------------
# multi val params are iterated by NewRound's, added to reports, start with column name.
+writer.version=LUCENE_31
merge.factor=mrg:10:100:10:100:10:100:10:100
max.buffered=buf:10:10:100:100:10:10:100:100
#ram.flush.mb=flush:32:40:48:56:32:40:48:56
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/standard.alg
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/standard.alg?rev=1031101&r1=1031100&r2=1031101&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/standard.alg (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/conf/standard.alg Thu Nov 4 17:49:19 2010
@@ -17,6 +17,7 @@
# -------------------------------------------------------------------------------------
# multi val params are iterated by NewRound's, added to reports, start with column name.
+writer.version=LUCENE_31
merge.factor=mrg:10:100:10:100:10:100:10:100
max.buffered=buf:10:10:100:100:10:10:100:100
compound=cmpnd:true:true:true:true:false:false:false:false
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java?rev=1031101&r1=1031100&r2=1031101&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java Thu Nov 4 17:49:19 2010
@@ -19,9 +19,12 @@ package org.apache.lucene.benchmark.byTa
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.utils.Config;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.MergeScheduler;
import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.MergePolicy;
@@ -29,6 +32,7 @@ import org.apache.lucene.index.NoDeletio
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.NoMergeScheduler;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Version;
import java.io.BufferedOutputStream;
@@ -69,21 +73,52 @@ public class CreateIndexTask extends Per
super(runData);
}
- public static void setIndexWriterConfig(IndexWriter writer, Config config) throws IOException {
+
+
+ public static IndexDeletionPolicy getIndexDeletionPolicy(Config config) {
+ String deletionPolicyName = config.get("deletion.policy", "org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy");
+ if (deletionPolicyName.equals(NoDeletionPolicy.class.getName())) {
+ return NoDeletionPolicy.INSTANCE;
+ } else {
+ try {
+ return Class.forName(deletionPolicyName).asSubclass(IndexDeletionPolicy.class).newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException("unable to instantiate class '" + deletionPolicyName + "' as IndexDeletionPolicy", e);
+ }
+ }
+ }
+
+ @Override
+ public int doLogic() throws IOException {
+ PerfRunData runData = getRunData();
+ Config config = runData.getConfig();
+ runData.setIndexWriter(configureWriter(config, runData, OpenMode.CREATE, null));
+ return 1;
+ }
+
+ public static IndexWriterConfig createWriterConfig(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit) {
+ Version version = Version.valueOf(config.get("writer.version", Version.LUCENE_31.toString()));
+ IndexWriterConfig iwConf = new IndexWriterConfig(version, runData.getAnalyzer());
+ iwConf.setOpenMode(mode);
+ IndexDeletionPolicy indexDeletionPolicy = getIndexDeletionPolicy(config);
+ iwConf.setIndexDeletionPolicy(indexDeletionPolicy);
+ if(commit != null)
+ iwConf.setIndexCommit(commit);
+
final String mergeScheduler = config.get("merge.scheduler",
"org.apache.lucene.index.ConcurrentMergeScheduler");
if (mergeScheduler.equals(NoMergeScheduler.class.getName())) {
- writer.setMergeScheduler(NoMergeScheduler.INSTANCE);
+ iwConf.setMergeScheduler(NoMergeScheduler.INSTANCE);
} else {
try {
- writer.setMergeScheduler(Class.forName(mergeScheduler).asSubclass(MergeScheduler.class).newInstance());
+ iwConf.setMergeScheduler(Class.forName(mergeScheduler).asSubclass(MergeScheduler.class).newInstance());
} catch (Exception e) {
throw new RuntimeException("unable to instantiate class '" + mergeScheduler + "' as merge scheduler", e);
}
if (mergeScheduler.equals("org.apache.lucene.index.ConcurrentMergeScheduler")) {
- ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler) writer.getMergeScheduler();
+ ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler) iwConf.getMergeScheduler();
int v = config.get("concurrent.merge.scheduler.max.thread.count", -1);
if (v != -1) {
cms.setMaxThreadCount(v);
@@ -99,28 +134,35 @@ public class CreateIndexTask extends Per
"org.apache.lucene.index.LogByteSizeMergePolicy");
boolean isCompound = config.get("compound", true);
if (mergePolicy.equals(NoMergePolicy.class.getName())) {
- writer.setMergePolicy(isCompound ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES);
+ iwConf.setMergePolicy(isCompound ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES);
} else {
try {
- writer.setMergePolicy(Class.forName(mergePolicy).asSubclass(MergePolicy.class).newInstance());
+ iwConf.setMergePolicy(Class.forName(mergePolicy).asSubclass(MergePolicy.class).newInstance());
} catch (Exception e) {
throw new RuntimeException("unable to instantiate class '" + mergePolicy + "' as merge policy", e);
}
- writer.setUseCompoundFile(isCompound);
- writer.setMergeFactor(config.get("merge.factor",OpenIndexTask.DEFAULT_MERGE_PFACTOR));
+ if(iwConf.getMergePolicy() instanceof LogMergePolicy) {
+ LogMergePolicy logMergePolicy = (LogMergePolicy) iwConf.getMergePolicy();
+ logMergePolicy.setUseCompoundFile(isCompound);
+ logMergePolicy.setMergeFactor(config.get("merge.factor",OpenIndexTask.DEFAULT_MERGE_PFACTOR));
+ }
}
- writer.setMaxFieldLength(config.get("max.field.length",OpenIndexTask.DEFAULT_MAX_FIELD_LENGTH));
-
+ iwConf.setMaxFieldLength(config.get("max.field.length",OpenIndexTask.DEFAULT_MAX_FIELD_LENGTH));
final double ramBuffer = config.get("ram.flush.mb",OpenIndexTask.DEFAULT_RAM_FLUSH_MB);
final int maxBuffered = config.get("max.buffered",OpenIndexTask.DEFAULT_MAX_BUFFERED);
if (maxBuffered == IndexWriterConfig.DISABLE_AUTO_FLUSH) {
- writer.setRAMBufferSizeMB(ramBuffer);
- writer.setMaxBufferedDocs(maxBuffered);
+ iwConf.setRAMBufferSizeMB(ramBuffer);
+ iwConf.setMaxBufferedDocs(maxBuffered);
} else {
- writer.setMaxBufferedDocs(maxBuffered);
- writer.setRAMBufferSizeMB(ramBuffer);
+ iwConf.setMaxBufferedDocs(maxBuffered);
+ iwConf.setRAMBufferSizeMB(ramBuffer);
}
+ return iwConf;
+ }
+
+ public static IndexWriter configureWriter(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit) throws CorruptIndexException, LockObtainFailedException, IOException {
+ IndexWriter writer = new IndexWriter(runData.getDirectory(), createWriterConfig(config, runData, mode, commit));
String infoStreamVal = config.get("writer.info.stream", null);
if (infoStreamVal != null) {
if (infoStreamVal.equals("SystemOut")) {
@@ -132,32 +174,6 @@ public class CreateIndexTask extends Per
writer.setInfoStream(new PrintStream(new BufferedOutputStream(new FileOutputStream(f))));
}
}
- }
-
- public static IndexDeletionPolicy getIndexDeletionPolicy(Config config) {
- String deletionPolicyName = config.get("deletion.policy", "org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy");
- if (deletionPolicyName.equals(NoDeletionPolicy.class.getName())) {
- return NoDeletionPolicy.INSTANCE;
- } else {
- try {
- return Class.forName(deletionPolicyName).asSubclass(IndexDeletionPolicy.class).newInstance();
- } catch (Exception e) {
- throw new RuntimeException("unable to instantiate class '" + deletionPolicyName + "' as IndexDeletionPolicy", e);
- }
- }
- }
-
- @Override
- public int doLogic() throws IOException {
- PerfRunData runData = getRunData();
- Config config = runData.getConfig();
-
- IndexWriter writer = new IndexWriter(runData.getDirectory(),
- new IndexWriterConfig(Version.LUCENE_31, runData.getAnalyzer())
- .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(
- getIndexDeletionPolicy(config)));
- setIndexWriterConfig(writer, config);
- runData.setIndexWriter(writer);
- return 1;
+ return writer;
}
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java?rev=1031101&r1=1031100&r2=1031101&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/OpenIndexTask.java Thu Nov 4 17:49:19 2010
@@ -23,8 +23,7 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LogMergePolicy;
-import org.apache.lucene.util.Version;
-
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import java.io.IOException;
@@ -62,11 +61,7 @@ public class OpenIndexTask extends PerfT
ic = null;
}
- IndexWriter writer = new IndexWriter(runData.getDirectory(),
- new IndexWriterConfig(Version.LUCENE_CURRENT, runData.getAnalyzer())
- .setIndexDeletionPolicy(CreateIndexTask.getIndexDeletionPolicy(config))
- .setIndexCommit(ic));
- CreateIndexTask.setIndexWriterConfig(writer, config);
+ final IndexWriter writer = CreateIndexTask.configureWriter(config, runData, OpenMode.APPEND, ic);
runData.setIndexWriter(writer);
return 1;
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTaskTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTaskTest.java?rev=1031101&r1=1031100&r2=1031101&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTaskTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTaskTest.java Thu Nov 4 17:49:19 2010
@@ -19,21 +19,29 @@ package org.apache.lucene.benchmark.byTa
import java.io.ByteArrayOutputStream;
import java.io.File;
+import java.io.IOException;
import java.io.PrintStream;
import java.util.Properties;
import org.apache.lucene.benchmark.BenchmarkTestCase;
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.utils.Config;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.NoDeletionPolicy;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.NoMergeScheduler;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.util.Version;
+
/** Tests the functionality of {@link CreateIndexTask}. */
public class CreateIndexTaskTest extends BenchmarkTestCase {
private PerfRunData createPerfRunData(String infoStreamValue) throws Exception {
Properties props = new Properties();
+ props.setProperty("writer.version", Version.LUCENE_31.toString());
props.setProperty("print.props", "false"); // don't print anything
props.setProperty("directory", "RAMDirectory");
if (infoStreamValue != null) {
@@ -102,5 +110,4 @@ public class CreateIndexTaskTest extends
new CreateIndexTask(runData).doLogic();
new CloseIndexTask(runData).doLogic();
}
-
}