You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by rm...@apache.org on 2010/01/12 21:06:19 UTC
svn commit: r898491 - in /lucene/java/trunk/contrib/benchmark: ./ conf/
scripts/ src/java/org/apache/lucene/benchmark/byTask/
src/java/org/apache/lucene/benchmark/byTask/tasks/
src/test/org/apache/lucene/benchmark/byTask/
Author: rmuir
Date: Tue Jan 12 20:06:17 2010
New Revision: 898491
URL: http://svn.apache.org/viewvc?rev=898491&view=rev
Log:
LUCENE-2181: add a benchmark for collation
Added:
lucene/java/trunk/contrib/benchmark/conf/collation.alg
lucene/java/trunk/contrib/benchmark/scripts/
lucene/java/trunk/contrib/benchmark/scripts/collation.bm2jira.pl (with props)
lucene/java/trunk/contrib/benchmark/scripts/compare.collation.benchmark.tables.pl (with props)
lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewCollationAnalyzerTask.java (with props)
lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewLocaleTask.java (with props)
Modified:
lucene/java/trunk/contrib/benchmark/CHANGES.txt
lucene/java/trunk/contrib/benchmark/build.xml
lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java
lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
Modified: lucene/java/trunk/contrib/benchmark/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/CHANGES.txt?rev=898491&r1=898490&r2=898491&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/CHANGES.txt (original)
+++ lucene/java/trunk/contrib/benchmark/CHANGES.txt Tue Jan 12 20:06:17 2010
@@ -4,6 +4,16 @@
$Id:$
+1/11/2010
+ LUCENE-2181: Add a benchmark for collation. This adds NewLocaleTask,
+ which sets a Locale in the run data for collation to use, and can be
+ used in the future for benchmarking localized range queries and sorts.
+ Also add NewCollationAnalyzerTask, which works with both JDK and ICU
+ Collator implementations. Fix ReadTokensTask to not tokenize fields
+ unless they should be tokenized according to DocMaker config. The
+ easiest way to run the benchmark is to run 'ant collation'
+ (Steven Rowe via Robert Muir)
+
12/22/2009
LUCENE-2178: Allow multiple locations to add to the class path with
-Dbenchmark.ext.classpath=... when running "ant run-task" (Steven
Modified: lucene/java/trunk/contrib/benchmark/build.xml
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/build.xml?rev=898491&r1=898490&r2=898491&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/build.xml (original)
+++ lucene/java/trunk/contrib/benchmark/build.xml Tue Jan 12 20:06:17 2010
@@ -24,7 +24,10 @@
<available file="temp/enwiki-20070527-pages-articles.xml.bz2" property="enwiki.exists"/>
<available file="temp/enwiki-20070527-pages-articles.xml" property="enwiki.expanded"/>
<available file="${working.dir}/enwiki.txt" property="enwiki.extracted"/>
-
+ <available file="temp/${top.100k.words.archive.filename}"
+ property="top.100k.words.archive.present"/>
+ <available file="${working.dir}/top100k-out"
+ property="top.100k.word.files.expanded"/>
</target>
<target name="enwiki-files" depends="check-files">
@@ -94,6 +97,27 @@
<untar src="temp/mini_newsgroups.tar" dest="${working.dir}"/>
</target>
+ <property name="top.100k.words.archive.filename"
+ value="top.100k.words.de.en.fr.uk.wikipedia.2009-11.tar.bz2"/>
+ <property name="top.100k.words.archive.base.url"
+ value="http://people.apache.org/~rmuir/wikipedia"/>
+ <target name="get-top-100k-words-archive" unless="top.100k.words.archive.present">
+ <mkdir dir="temp"/>
+ <get src="${top.100k.words.archive.base.url}/${top.100k.words.archive.filename}"
+ dest="temp/${top.100k.words.archive.filename}"/>
+ </target>
+ <target name="expand-top-100k-word-files" unless="top.100k.word.files.expanded">
+ <mkdir dir="${working.dir}/top100k-out"/>
+ <untar src="temp/${top.100k.words.archive.filename}"
+ overwrite="true" compression="bzip2" dest="${working.dir}/top100k-out"/>
+ </target>
+
+ <target name="top-100k-wiki-word-files" depends="check-files">
+ <mkdir dir="${working.dir}"/>
+ <antcall target="get-top-100k-words-archive"/>
+ <antcall target="expand-top-100k-word-files"/>
+ </target>
+
<target name="get-files" depends="check-files">
<mkdir dir="temp"/>
<antcall target="get-reuters"/>
@@ -141,6 +165,34 @@
</java>
</target>
+ <property name="collation.alg.file" location="conf/collation.alg"/>
+ <property name="collation.output.file"
+ value="${working.dir}/collation.benchmark.output.txt"/>
+ <property name="collation.jira.output.file"
+ value="${working.dir}/collation.bm2jira.output.txt"/>
+
+ <path id="collation.runtime.classpath">
+ <path refid="run.classpath"/>
+ <pathelement path="${common.dir}/build/contrib/icu/classes/java"/>
+ <fileset dir="${common.dir}/contrib/icu/lib" includes="icu4j*.jar"/>
+ </path>
+
+ <target name="collation" depends="compile,compile-icu,top-100k-wiki-word-files">
+ <echo>Running contrib/benchmark with alg file: ${collation.alg.file}</echo>
+ <java fork="true" classname="org.apache.lucene.benchmark.byTask.Benchmark"
+ maxmemory="${task.mem}" output="${collation.output.file}">
+ <classpath refid="collation.runtime.classpath"/>
+ <arg file="${collation.alg.file}"/>
+ </java>
+ <echo>Benchmark output is in file: ${collation.output.file}</echo>
+ <echo>Converting to JIRA table format...</echo>
+ <exec executable="perl" output="${collation.jira.output.file}" failonerror="true">
+ <arg value="scripts/collation.bm2jira.pl"/>
+ <arg value="${collation.output.file}"/>
+ </exec>
+ <echo>Benchmark output in JIRA table format is in file: ${collation.jira.output.file}</echo>
+ </target>
+
<target name="compile-demo">
<subant target="compile-demo">
<fileset dir="${common.dir}" includes="build.xml"/>
@@ -151,6 +203,11 @@
<fileset dir="${common.dir}/contrib/highlighter" includes="build.xml"/>
</subant>
</target>
+ <target name="compile-icu">
+ <subant target="compile">
+ <fileset dir="${common.dir}/contrib/icu" includes="build.xml"/>
+ </subant>
+ </target>
<target name="compile-memory">
<subant target="compile">
<fileset dir="${common.dir}/contrib/memory" includes="build.xml"/>
Added: lucene/java/trunk/contrib/benchmark/conf/collation.alg
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/conf/collation.alg?rev=898491&view=auto
==============================================================================
--- lucene/java/trunk/contrib/benchmark/conf/collation.alg (added)
+++ lucene/java/trunk/contrib/benchmark/conf/collation.alg Tue Jan 12 20:06:17 2010
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource
+content.source.encoding=UTF-8
+doc.tokenized=false
+doc.body.tokenized=true
+docs.file=work/top100k-out/top.fr.wikipedia.words.txt
+content.source.forever=false
+log.step=100000
+
+{ "Rounds"
+ -NewAnalyzer(KeywordAnalyzer)
+ -SetProp(docs.file,work/top100k-out/top.fr.wikipedia.words.txt)
+ ResetInputs
+ { "FrenchKeyword" { ReadTokens > : * ResetInputs } : 10
+
+ -NewAnalyzer(KeywordAnalyzer)
+ -SetProp(docs.file,work/top100k-out/top.de.wikipedia.words.txt)
+ ResetInputs
+ { "GermanKeyword" { ReadTokens > : * ResetInputs } : 10
+
+ -NewAnalyzer(KeywordAnalyzer)
+ -SetProp(docs.file,work/top100k-out/top.uk.wikipedia.words.txt)
+ ResetInputs
+ { "UkrainianKeyword" { ReadTokens > : * ResetInputs } : 10
+
+ -NewAnalyzer(KeywordAnalyzer)
+ -SetProp(docs.file,work/top100k-out/top.en.wikipedia.words.txt)
+ ResetInputs
+ { "EnglishKeyword" { ReadTokens > : * ResetInputs } : 10
+
+ -NewLocale(fr)
+ -NewCollationAnalyzer
+ -SetProp(docs.file,work/top100k-out/top.fr.wikipedia.words.txt)
+ ResetInputs
+ { "FrenchJDK" { ReadTokens > : * ResetInputs } : 10
+
+ -NewLocale(de)
+ -NewCollationAnalyzer
+ -SetProp(docs.file,work/top100k-out/top.de.wikipedia.words.txt)
+ ResetInputs
+ { "GermanJDK" { ReadTokens > : * ResetInputs } : 10
+
+ -NewLocale(uk)
+ -NewCollationAnalyzer
+ -SetProp(docs.file,work/top100k-out/top.uk.wikipedia.words.txt)
+ ResetInputs
+ { "UkrainianJDK" { ReadTokens > : * ResetInputs } : 10
+
+ -NewLocale(en)
+ -NewCollationAnalyzer
+ -SetProp(docs.file,work/top100k-out/top.en.wikipedia.words.txt)
+ ResetInputs
+ { "EnglishJDK" { ReadTokens > : * ResetInputs } : 10
+
+ -NewLocale(fr)
+ -NewCollationAnalyzer(impl:icu)
+ -SetProp(docs.file,work/top100k-out/top.fr.wikipedia.words.txt)
+ ResetInputs
+ { "FrenchICU" { ReadTokens > : * ResetInputs } : 10
+
+ -NewLocale(de)
+ -NewCollationAnalyzer(impl:icu)
+ -SetProp(docs.file,work/top100k-out/top.de.wikipedia.words.txt)
+ ResetInputs
+ { "GermanICU" { ReadTokens > : * ResetInputs } : 10
+
+ -NewLocale(uk)
+ -NewCollationAnalyzer(impl:icu)
+ -SetProp(docs.file,work/top100k-out/top.uk.wikipedia.words.txt)
+ ResetInputs
+ { "UkrainianICU" { ReadTokens > : * ResetInputs } : 10
+
+ -NewLocale(en)
+ -NewCollationAnalyzer(impl:icu)
+ -SetProp(docs.file,work/top100k-out/top.en.wikipedia.words.txt)
+ ResetInputs
+ { "EnglishICU" { ReadTokens > : * ResetInputs } : 10
+
+ NewRound
+
+} : 5
+
+RepSumByNameRound
Added: lucene/java/trunk/contrib/benchmark/scripts/collation.bm2jira.pl
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/scripts/collation.bm2jira.pl?rev=898491&view=auto
==============================================================================
--- lucene/java/trunk/contrib/benchmark/scripts/collation.bm2jira.pl (added)
+++ lucene/java/trunk/contrib/benchmark/scripts/collation.bm2jira.pl Tue Jan 12 20:06:17 2010
@@ -0,0 +1,63 @@
+#!/usr/bin/perl
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ----------
+# bm2jira.pl
+#
+# Converts Lucene contrib-benchmark output produced using the
+# benchmark.collation.alg file into a JIRA-formatted table.
+#
+
+use strict;
+use warnings;
+
+my %min_elapsed = ();
+
+while (<>) {
+ if (/(\S+)(Keyword|JDK|ICU)_\d+\s*([^\s{].*)/) {
+ my $lang = $1;
+ my $analyzer = $2;
+ my $stats = $3;
+ my ($elapsed) = $stats =~ /(?:[\d,.]+[-\s]*){4}([.\d]+)/;
+ $min_elapsed{$analyzer}{$lang} = $elapsed
+ unless (defined($min_elapsed{$analyzer}{$lang})
+ && $elapsed >= $min_elapsed{$analyzer}{$lang});
+ }
+}
+
+# Print out platform info
+print "JAVA:\n", `java -version 2>&1`, "\nOS:\n";
+if ($^O =~ /win/i) {
+ print "$^O\n";
+ eval {
+ require Win32;
+ print Win32::GetOSName(), "\n", Win32::GetOSVersion(), "\n";
+ };
+ die "Error loading Win32: $@" if ($@);
+} else {
+ print `uname -a 2>&1`;
+}
+
+print "\n||Language||java.text||ICU4J||KeywordAnalyzer||ICU4J Improvement||\n";
+
+for my $lang (sort keys %{$min_elapsed{ICU}}) {
+ my $ICU = $min_elapsed{ICU}{$lang};
+ my $JDK = $min_elapsed{JDK}{$lang};
+ my $keyword = $min_elapsed{Keyword}{$lang};
+ my $improved = int(100 * ($JDK - $ICU) / ($ICU - $keyword) + 0.5);
+ printf "|$lang|${JDK}s|${ICU}s|${keyword}s|\%d%%|\n", $improved;
+}
Propchange: lucene/java/trunk/contrib/benchmark/scripts/collation.bm2jira.pl
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/java/trunk/contrib/benchmark/scripts/compare.collation.benchmark.tables.pl
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/scripts/compare.collation.benchmark.tables.pl?rev=898491&view=auto
==============================================================================
--- lucene/java/trunk/contrib/benchmark/scripts/compare.collation.benchmark.tables.pl (added)
+++ lucene/java/trunk/contrib/benchmark/scripts/compare.collation.benchmark.tables.pl Tue Jan 12 20:06:17 2010
@@ -0,0 +1,91 @@
+#!/usr/bin/perl
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ------------------------------------------
+# compare.collation.benchmark.jira.tables.pl
+#
+# Takes as cmdline parameters two JIRA-formatted benchmark results, as produced
+# by bm2jira.pl (located in the same directory as this script), and outputs a
+# third JIRA-formatted comparison table, showing the differences between two
+# benchmarking runs' java.text and ICU4J columns, after accounting for the
+# KeywordAnalyzer column; the "ICU4J Improvement" column is ignored.
+#
+# The difference is calculated as a percentage:
+#
+# 100 * (patched-rate - unpatched-rate / unpatched-rate)
+#
+# where the (un)patched-rate is:
+#
+# 1 / ( elapsed-(un)patched-time - elapsed-KeywordAnalyzer-time)
+#
+
+use strict;
+use warnings;
+
+my $usage = "Usage: $0 <unpatched-file> <patched-file>\n";
+
+die $usage unless ($#ARGV == 1 && -f $ARGV[0] && -f $ARGV[1]);
+
+my %stats = ();
+
+open UNPATCHED, "<$ARGV[0]" || die "ERROR opening '$ARGV[0]': $!";
+while (<UNPATCHED>) {
+ # ||Language||java.text||ICU4J||KeywordAnalyzer||ICU4J Improvement||
+ # |English|4.51s|2.47s|1.47s|204%|
+ next unless (/^\|([^|]+)\|([^|s]+)s\|([^|s]+)s\|([^|s]+)s/);
+ my ($lang, $jdk_elapsed, $icu_elapsed, $keyword_analyzer_elapsed)
+ = ($1, $2, $3, $4);
+ $stats{unpatched}{$lang}{jdk} = $jdk_elapsed;
+ $stats{unpatched}{$lang}{icu} = $icu_elapsed;
+ $stats{unpatched}{$lang}{keyword_analyzer} = $keyword_analyzer_elapsed;
+}
+close UNPATCHED;
+
+open PATCHED, "<$ARGV[1]" || die "ERROR opening '$ARGV[1]': $!";
+while (<PATCHED>) {
+ # ||Language||java.text||ICU4J||KeywordAnalyzer||ICU4J Improvement||
+ # |English|4.51s|2.47s|1.47s|204%|
+ next unless (/^\|([^|]+)\|([^|s]+)s\|([^|s]+)s\|([^|s]+)s/);
+ my ($lang, $jdk_elapsed, $icu_elapsed, $keyword_analyzer_elapsed)
+ = ($1, $2, $3, $4);
+ $stats{patched}{$lang}{jdk} = $jdk_elapsed;
+ $stats{patched}{$lang}{icu} = $icu_elapsed;
+ $stats{patched}{$lang}{keyword_analyzer} = $keyword_analyzer_elapsed;
+}
+close PATCHED;
+
+print "||Language||java.text improvement||ICU4J improvement||\n";
+for my $lang (sort keys %{$stats{unpatched}}) {
+ my $keyword_analyzer1 = $stats{unpatched}{$lang}{keyword_analyzer};
+ my $jdk1 = $stats{unpatched}{$lang}{jdk};
+ my $jdk_diff1 = $jdk1 - $keyword_analyzer1;
+ my $icu1 = $stats{unpatched}{$lang}{icu};
+ my $icu_diff1 = $icu1 - $keyword_analyzer1;
+
+ my $keyword_analyzer2 = $stats{patched}{$lang}{keyword_analyzer};
+ my $jdk2 = $stats{patched}{$lang}{jdk};
+ my $jdk_diff2 = $jdk2 - $keyword_analyzer2;
+ my $icu2 = $stats{patched}{$lang}{icu};
+ my $icu_diff2 = $icu2 - $keyword_analyzer2;
+
+ my $jdk_impr
+ = int((1./$jdk_diff2 - 1./$jdk_diff1) / (1./$jdk_diff1) * 1000 + 5) / 10;
+ my $icu_impr
+ = int((1./$icu_diff2 - 1./$icu_diff1) / (1./$icu_diff1) * 1000 + 5) / 10;
+
+ printf "|$lang|%2.1f%%|%2.1f%%|\n", $jdk_impr, $icu_impr;
+}
Propchange: lucene/java/trunk/contrib/benchmark/scripts/compare.collation.benchmark.tables.pl
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java?rev=898491&r1=898490&r2=898491&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java Tue Jan 12 20:06:17 2010
@@ -20,6 +20,7 @@
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
+import java.util.Locale;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
@@ -61,6 +62,7 @@
private Directory directory;
private Analyzer analyzer;
private DocMaker docMaker;
+ private Locale locale;
// we use separate (identical) instances for each "read" task type, so each can iterate the quries separately.
private HashMap<Class<? extends ReadTask>,QueryMaker> readTaskQueryMaker;
@@ -245,6 +247,20 @@
}
/**
+ * @return the locale
+ */
+ public Locale getLocale() {
+ return locale;
+ }
+
+ /**
+ * @param locale the locale to set
+ */
+ public void setLocale(Locale locale) {
+ this.locale = locale;
+ }
+
+ /**
* @return Returns the config.
*/
public Config getConfig() {
Added: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewCollationAnalyzerTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewCollationAnalyzerTask.java?rev=898491&view=auto
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewCollationAnalyzerTask.java (added)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewCollationAnalyzerTask.java Tue Jan 12 20:06:17 2010
@@ -0,0 +1,117 @@
+package org.apache.lucene.benchmark.byTask.tasks;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.util.Locale;
+import java.util.StringTokenizer;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+
+/**
+ * Task to support benchmarking collation.
+ * <p>
+ * <ul>
+ * <li> <code>NewCollationAnalyzer</code> with the default jdk impl
+ * <li> <code>NewCollationAnalyzer(impl:icu)</code> specify an impl (jdk,icu)
+ * </ul>
+ * </p>
+ */
+public class NewCollationAnalyzerTask extends PerfTask {
+ public enum Implementation {
+ JDK("org.apache.lucene.collation.CollationKeyAnalyzer",
+ "java.text.Collator"),
+ ICU("org.apache.lucene.collation.ICUCollationKeyAnalyzer",
+ "com.ibm.icu.text.Collator");
+
+ String className;
+ String collatorClassName;
+
+ Implementation(String className, String collatorClassName) {
+ this.className = className;
+ this.collatorClassName = collatorClassName;
+ }
+ }
+
+ private Implementation impl = Implementation.JDK;
+
+ public NewCollationAnalyzerTask(PerfRunData runData) {
+ super(runData);
+ }
+
+ static Analyzer createAnalyzer(Locale locale, Implementation impl)
+ throws Exception {
+ final Class<?> collatorClazz = Class.forName(impl.collatorClassName);
+ Method collatorMethod = collatorClazz.getMethod("getInstance",
+ new Class[] {Locale.class});
+ Object collator = collatorMethod.invoke(null, locale);
+
+ final Class<? extends Analyzer> clazz = Class.forName(impl.className)
+ .asSubclass(Analyzer.class);
+ Constructor<? extends Analyzer> ctor = clazz.getConstructor(collatorClazz);
+ return ctor.newInstance(collator);
+ }
+
+ @Override
+ public int doLogic() throws Exception {
+ try {
+ Locale locale = getRunData().getLocale();
+ if (locale == null) throw new RuntimeException(
+ "Locale must be set with the NewLocale task!");
+ Analyzer analyzer = createAnalyzer(locale, impl);
+ getRunData().setAnalyzer(analyzer);
+ System.out.println("Changed Analyzer to: "
+ + analyzer.getClass().getName() + "(" + locale + ")");
+ } catch (Exception e) {
+ throw new RuntimeException("Error creating Analyzer: impl=" + impl, e);
+ }
+ return 1;
+ }
+
+ @Override
+ public void setParams(String params) {
+ super.setParams(params);
+
+ StringTokenizer st = new StringTokenizer(params, ",");
+ while (st.hasMoreTokens()) {
+ String param = st.nextToken();
+ StringTokenizer expr = new StringTokenizer(param, ":");
+ String key = expr.nextToken();
+ String value = expr.nextToken();
+ // for now we only support the "impl" parameter.
+ // TODO: add strength, decomposition, etc
+ if (key.equals("impl")) {
+ if (value.equalsIgnoreCase("icu"))
+ impl = Implementation.ICU;
+ else if (value.equalsIgnoreCase("jdk"))
+ impl = Implementation.JDK;
+ else
+ throw new RuntimeException("Unknown parameter " + param);
+ } else {
+ throw new RuntimeException("Unknown parameter " + param);
+ }
+ }
+ }
+
+ @Override
+ public boolean supportsParams() {
+ return true;
+ }
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewCollationAnalyzerTask.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewLocaleTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewLocaleTask.java?rev=898491&view=auto
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewLocaleTask.java (added)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewLocaleTask.java Tue Jan 12 20:06:17 2010
@@ -0,0 +1,89 @@
+package org.apache.lucene.benchmark.byTask.tasks;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Locale;
+import java.util.StringTokenizer;
+
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+
+/**
+ * Set a {@link java.util.Locale} for use in benchmarking.
+ * <p>
+ * Locales can be specified in the following ways:
+ * <ul>
+ * <li><code>de</code>: Language "de"
+ * <li><code>en,US</code>: Language "en", country "US"
+ * <li><code>no,NO,NY</code>: Language "no", country "NO", variant "NY"
+ * <li><code>ROOT</code>: The root (language-agnostic) Locale
+ * <li><empty string>: Erase the Locale (null)
+ * </ul>
+ * </p>
+ */
+public class NewLocaleTask extends PerfTask {
+ private String language;
+ private String country;
+ private String variant;
+
+ /**
+ * Create a new {@link java.util.Locale} and set it it in the getRunData() for
+ * use by all future tasks.
+ */
+ public NewLocaleTask(PerfRunData runData) {
+ super(runData);
+ }
+
+ static Locale createLocale(String language, String country, String variant) {
+ if (language == null || language.length() == 0)
+ return null;
+
+ String lang = language;
+ if (lang.equalsIgnoreCase("ROOT"))
+ lang = ""; // empty language is the root locale in the JDK
+
+ return new Locale(lang, country, variant);
+ }
+
+ @Override
+ public int doLogic() throws Exception {
+ Locale locale = createLocale(language, country, variant);
+ getRunData().setLocale(locale);
+ System.out.println("Changed Locale to: " +
+ (locale == null ? "null" :
+ (locale.getDisplayName().length() == 0) ? "root locale" : locale));
+ return 1;
+ }
+
+ @Override
+ public void setParams(String params) {
+ super.setParams(params);
+ language = country = variant = "";
+ StringTokenizer st = new StringTokenizer(params, ",");
+ if (st.hasMoreTokens())
+ language = st.nextToken();
+ if (st.hasMoreTokens())
+ country = st.nextToken();
+ if (st.hasMoreTokens())
+ variant = st.nextToken();
+ }
+
+ @Override
+ public boolean supportsParams() {
+ return true;
+ }
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewLocaleTask.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java?rev=898491&r1=898490&r2=898491&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java Tue Jan 12 20:06:17 2010
@@ -67,6 +67,8 @@
Analyzer analyzer = getRunData().getAnalyzer();
int tokenCount = 0;
for(final Fieldable field : fields) {
+ if (!field.isTokenized()) continue;
+
final TokenStream stream;
final TokenStream streamValue = field.tokenStreamValue();
Modified: lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=898491&r1=898490&r2=898491&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Tue Jan 12 20:06:17 2010
@@ -21,15 +21,23 @@
import java.io.File;
import java.io.FileReader;
import java.io.BufferedReader;
+import java.text.Collator;
import java.util.List;
import java.util.Iterator;
+import java.util.Locale;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
import org.apache.lucene.benchmark.byTask.feeds.ReutersQueryMaker;
import org.apache.lucene.benchmark.byTask.tasks.CountingSearchTestTask;
import org.apache.lucene.benchmark.byTask.tasks.CountingHighlighterTestTask;
import org.apache.lucene.benchmark.byTask.stats.TaskStats;
+import org.apache.lucene.collation.CollationKeyAnalyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermEnum;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.SerialMergeScheduler;
@@ -464,9 +472,13 @@
TermDocs termDocs = reader.termDocs();
int totalTokenCount2 = 0;
while(terms.next()) {
- termDocs.seek(terms.term());
- while(termDocs.next())
- totalTokenCount2 += termDocs.freq();
+ Term term = terms.term();
+ /* not-tokenized, but indexed field */
+ if (term != null && term.field() != DocMaker.ID_FIELD) {
+ termDocs.seek(terms.term());
+ while (termDocs.next())
+ totalTokenCount2 += termDocs.freq();
+ }
}
reader.close();
@@ -850,6 +862,119 @@
};
}
+ /**
+ * Test that we can change the Locale in the runData,
+ * that it is parsed as we expect.
+ */
+ public void testLocale() throws Exception {
+ // empty Locale: clear it (null)
+ Benchmark benchmark = execBenchmark(getLocaleConfig(""));
+ assertNull(benchmark.getRunData().getLocale());
+
+ // ROOT locale
+ benchmark = execBenchmark(getLocaleConfig("ROOT"));
+ assertEquals(new Locale(""), benchmark.getRunData().getLocale());
+
+ // specify just a language
+ benchmark = execBenchmark(getLocaleConfig("de"));
+ assertEquals(new Locale("de"), benchmark.getRunData().getLocale());
+
+ // specify language + country
+ benchmark = execBenchmark(getLocaleConfig("en,US"));
+ assertEquals(new Locale("en", "US"), benchmark.getRunData().getLocale());
+
+ // specify language + country + variant
+ benchmark = execBenchmark(getLocaleConfig("no,NO,NY"));
+ assertEquals(new Locale("no", "NO", "NY"), benchmark.getRunData().getLocale());
+ }
+
+ private static String[] getLocaleConfig(String localeParam) {
+ String algLines[] = {
+ "# ----- properties ",
+ "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.log.step=3",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "# ----- alg ",
+ "{ \"Rounds\"",
+ " ResetSystemErase",
+ " NewLocale(" + localeParam + ")",
+ " CreateIndex",
+ " { \"AddDocs\" AddDoc > : * ",
+ " NewRound",
+ "} : 1",
+ };
+ return algLines;
+ }
+
+ /**
+ * Test that we can create CollationAnalyzers.
+ */
+ public void testCollator() throws Exception {
+ // ROOT locale
+ Benchmark benchmark = execBenchmark(getCollatorConfig("ROOT", "impl:jdk"));
+ CollationKeyAnalyzer expected = new CollationKeyAnalyzer(Collator
+ .getInstance(new Locale("")));
+ assertEqualCollation(expected, benchmark.getRunData().getAnalyzer(), "foobar");
+
+ // specify just a language
+ benchmark = execBenchmark(getCollatorConfig("de", "impl:jdk"));
+ expected = new CollationKeyAnalyzer(Collator.getInstance(new Locale("de")));
+ assertEqualCollation(expected, benchmark.getRunData().getAnalyzer(), "foobar");
+
+ // specify language + country
+ benchmark = execBenchmark(getCollatorConfig("en,US", "impl:jdk"));
+ expected = new CollationKeyAnalyzer(Collator.getInstance(new Locale("en",
+ "US")));
+ assertEqualCollation(expected, benchmark.getRunData().getAnalyzer(), "foobar");
+
+ // specify language + country + variant
+ benchmark = execBenchmark(getCollatorConfig("no,NO,NY", "impl:jdk"));
+ expected = new CollationKeyAnalyzer(Collator.getInstance(new Locale("no",
+ "NO", "NY")));
+ assertEqualCollation(expected, benchmark.getRunData().getAnalyzer(), "foobar");
+ }
+
+ private void assertEqualCollation(Analyzer a1, Analyzer a2, String text)
+ throws Exception {
+ TokenStream ts1 = a1.tokenStream("bogus", new StringReader(text));
+ TokenStream ts2 = a2.tokenStream("bogus", new StringReader(text));
+ ts1.reset();
+ ts2.reset();
+ TermAttribute termAtt1 = ts1.addAttribute(TermAttribute.class);
+ TermAttribute termAtt2 = ts2.addAttribute(TermAttribute.class);
+ assertTrue(ts1.incrementToken());
+ assertTrue(ts2.incrementToken());
+ assertEquals(termAtt1.term(), termAtt2.term());
+ assertFalse(ts1.incrementToken());
+ assertFalse(ts2.incrementToken());
+ ts1.close();
+ ts2.close();
+ }
+
+ private static String[] getCollatorConfig(String localeParam,
+ String collationParam) {
+ String algLines[] = {
+ "# ----- properties ",
+ "content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource",
+ "docs.file=" + getReuters20LinesFile(),
+ "content.source.log.step=3",
+ "content.source.forever=false",
+ "directory=RAMDirectory",
+ "# ----- alg ",
+ "{ \"Rounds\"",
+ " ResetSystemErase",
+ " NewLocale(" + localeParam + ")",
+ " NewCollationAnalyzer(" + collationParam + ")",
+ " CreateIndex",
+ " { \"AddDocs\" AddDoc > : * ",
+ " NewRound",
+ "} : 1",
+ };
+ return algLines;
+ }
+
private static String getReuters20LinesFile() {
return System.getProperty("lucene.common.dir").replace('\\','/') +
"/contrib/benchmark/src/test/org/apache/lucene/benchmark/reuters.first20.lines.txt";