You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by ot...@apache.org on 2007/04/03 01:37:17 UTC
svn commit: r524969 [2/5] - in
/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark:
./ byTask/ byTask/feeds/ byTask/programmatic/ byTask/stats/ byTask/tasks/
byTask/utils/ standard/ stats/ utils/
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleQueryMaker.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleQueryMaker.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleQueryMaker.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleQueryMaker.java Mon Apr 2 16:37:14 2007
@@ -1,68 +1,68 @@
-package org.apache.lucene.benchmark.byTask.feeds;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.QueryParser;
+package org.apache.lucene.benchmark.byTask.feeds;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+
import java.util.ArrayList;
-/**
- * A QueryMaker that makes queries for a collection created
- * using {@link org.apache.lucene.benchmark.byTask.feeds.SimpleDocMaker}.
- */
+/**
+ * A QueryMaker that makes queries for a collection created
+ * using {@link org.apache.lucene.benchmark.byTask.feeds.SimpleDocMaker}.
+ */
public class SimpleQueryMaker extends AbstractQueryMaker implements QueryMaker {
-
- /**
- * Prepare the queries for this test.
- * Extending classes can overide this method for preparing different queries.
- * @return prepared queries.
- * @throws Exception if canot prepare the queries.
- */
- protected Query[] prepareQueries() throws Exception {
- // analyzer (default is standard analyzer)
- Analyzer anlzr= (Analyzer) Class.forName(config.get("analyzer",
- "org.apache.lucene.analysis.standard.StandardAnalyzer")).newInstance();
-
- QueryParser qp = new QueryParser("body",anlzr);
- ArrayList qq = new ArrayList();
- Query q1 = new TermQuery(new Term("docid","doc2"));
- qq.add(q1);
- Query q2 = new TermQuery(new Term("body","simple"));
- qq.add(q2);
- BooleanQuery bq = new BooleanQuery();
- bq.add(q1,Occur.MUST);
- bq.add(q2,Occur.MUST);
- qq.add(bq);
- qq.add(qp.parse("synthetic body"));
- qq.add(qp.parse("\"synthetic body\""));
- qq.add(qp.parse("synthetic text"));
- qq.add(qp.parse("\"synthetic text\""));
- qq.add(qp.parse("\"synthetic text\"~3"));
- qq.add(qp.parse("zoom*"));
- qq.add(qp.parse("synth*"));
- return (Query []) qq.toArray(new Query[0]);
- }
-
-}
+
+ /**
+ * Prepare the queries for this test.
+ * Extending classes can overide this method for preparing different queries.
+ * @return prepared queries.
+ * @throws Exception if canot prepare the queries.
+ */
+ protected Query[] prepareQueries() throws Exception {
+ // analyzer (default is standard analyzer)
+ Analyzer anlzr= (Analyzer) Class.forName(config.get("analyzer",
+ "org.apache.lucene.analysis.standard.StandardAnalyzer")).newInstance();
+
+ QueryParser qp = new QueryParser("body",anlzr);
+ ArrayList qq = new ArrayList();
+ Query q1 = new TermQuery(new Term("docid","doc2"));
+ qq.add(q1);
+ Query q2 = new TermQuery(new Term("body","simple"));
+ qq.add(q2);
+ BooleanQuery bq = new BooleanQuery();
+ bq.add(q1,Occur.MUST);
+ bq.add(q2,Occur.MUST);
+ qq.add(bq);
+ qq.add(qp.parse("synthetic body"));
+ qq.add(qp.parse("\"synthetic body\""));
+ qq.add(qp.parse("synthetic text"));
+ qq.add(qp.parse("\"synthetic text\""));
+ qq.add(qp.parse("\"synthetic text\"~3"));
+ qq.add(qp.parse("zoom*"));
+ qq.add(qp.parse("synth*"));
+ return (Query []) qq.toArray(new Query[0]);
+ }
+
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleQueryMaker.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleSloppyPhraseQueryMaker.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleSloppyPhraseQueryMaker.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleSloppyPhraseQueryMaker.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleSloppyPhraseQueryMaker.java Mon Apr 2 16:37:14 2007
@@ -1,65 +1,65 @@
-package org.apache.lucene.benchmark.byTask.feeds;
-
-import java.util.ArrayList;
-import java.util.StringTokenizer;
-
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.Query;
-
-/**
- * Create sloppy phrase queries for performance test, in an index created using simple doc maker.
- */
-public class SimpleSloppyPhraseQueryMaker extends SimpleQueryMaker {
-
- /* (non-Javadoc)
- * @see org.apache.lucene.benchmark.byTask.feeds.SimpleQueryMaker#prepareQueries()
- */
- protected Query[] prepareQueries() throws Exception {
- // exatract some 100 words from doc text to an array
- String words[];
- ArrayList w = new ArrayList();
- StringTokenizer st = new StringTokenizer(SimpleDocMaker.DOC_TEXT);
- while (st.hasMoreTokens() && w.size()<100) {
- w.add(st.nextToken());
- }
- words = (String[]) w.toArray(new String[0]);
-
- // create queries (that would find stuff) with varying slops
- ArrayList queries = new ArrayList();
- for (int slop=0; slop<8; slop++) {
- for (int qlen=2; qlen<6; qlen++) {
- for (int wd=0; wd<words.length-qlen-slop; wd++) {
- // ordered
- int remainedSlop = slop;
- PhraseQuery q = new PhraseQuery();
- q.setSlop(slop);
- int wind = wd;
- for (int i=0; i<qlen; i++) {
+package org.apache.lucene.benchmark.byTask.feeds;
+
+import java.util.ArrayList;
+import java.util.StringTokenizer;
+
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.Query;
+
+/**
+ * Create sloppy phrase queries for performance test, in an index created using simple doc maker.
+ */
+public class SimpleSloppyPhraseQueryMaker extends SimpleQueryMaker {
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.benchmark.byTask.feeds.SimpleQueryMaker#prepareQueries()
+ */
+ protected Query[] prepareQueries() throws Exception {
+ // exatract some 100 words from doc text to an array
+ String words[];
+ ArrayList w = new ArrayList();
+ StringTokenizer st = new StringTokenizer(SimpleDocMaker.DOC_TEXT);
+ while (st.hasMoreTokens() && w.size()<100) {
+ w.add(st.nextToken());
+ }
+ words = (String[]) w.toArray(new String[0]);
+
+ // create queries (that would find stuff) with varying slops
+ ArrayList queries = new ArrayList();
+ for (int slop=0; slop<8; slop++) {
+ for (int qlen=2; qlen<6; qlen++) {
+ for (int wd=0; wd<words.length-qlen-slop; wd++) {
+ // ordered
+ int remainedSlop = slop;
+ PhraseQuery q = new PhraseQuery();
+ q.setSlop(slop);
+ int wind = wd;
+ for (int i=0; i<qlen; i++) {
q.add(new Term(BasicDocMaker.BODY_FIELD,words[wind++]));
- if (remainedSlop>0) {
- remainedSlop--;
- wind++;
- }
- }
- queries.add(q);
- // reveresed
- remainedSlop = slop;
- q = new PhraseQuery();
- q.setSlop(slop+2*qlen);
- wind = wd+qlen+remainedSlop-1;
- for (int i=0; i<qlen; i++) {
+ if (remainedSlop>0) {
+ remainedSlop--;
+ wind++;
+ }
+ }
+ queries.add(q);
+ // reveresed
+ remainedSlop = slop;
+ q = new PhraseQuery();
+ q.setSlop(slop+2*qlen);
+ wind = wd+qlen+remainedSlop-1;
+ for (int i=0; i<qlen; i++) {
q.add(new Term(BasicDocMaker.BODY_FIELD,words[wind--]));
- if (remainedSlop>0) {
- remainedSlop--;
- wind--;
- }
- }
- queries.add(q);
- }
- }
- }
- return (Query[]) queries.toArray(new Query[0]);
- }
-
-}
+ if (remainedSlop>0) {
+ remainedSlop--;
+ wind--;
+ }
+ }
+ queries.add(q);
+ }
+ }
+ }
+ return (Query[]) queries.toArray(new Query[0]);
+ }
+
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SimpleSloppyPhraseQueryMaker.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocMaker.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocMaker.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocMaker.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocMaker.java Mon Apr 2 16:37:14 2007
@@ -1,221 +1,221 @@
-package org.apache.lucene.benchmark.byTask.feeds;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.Locale;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.lucene.benchmark.byTask.utils.Config;
-
-
-/**
- * A DocMaker using the (compressed) Trec collection for its input.
- */
-public class TrecDocMaker extends BasicDocMaker {
-
- private static final String newline = System.getProperty("line.separator");
-
- private DateFormat dateFormat [];
- private File dataDir = null;
- private ArrayList inputFiles = new ArrayList();
- private int nextFile = 0;
- private int iteration=0;
- private BufferedReader reader;
- private GZIPInputStream zis;
-
- private static final String DATE_FORMATS [] = {
- "EEE, dd MMM yyyy kk:mm:ss z", //Tue, 09 Dec 2003 22:39:08 GMT
- "EEE MMM dd kk:mm:ss yyyy z", //Tue Dec 09 16:45:08 2003 EST
- "EEE, dd-MMM-':'y kk:mm:ss z", //Tue, 09 Dec 2003 22:39:08 GMT
- "EEE, dd-MMM-yyy kk:mm:ss z", //Tue, 09 Dec 2003 22:39:08 GMT
- };
-
- /* (non-Javadoc)
- * @see SimpleDocMaker#setConfig(java.util.Properties)
- */
- public void setConfig(Config config) {
- super.setConfig(config);
- String d = config.get("docs.dir","trec");
- dataDir = new File(new File("work"),d);
- collectFiles(dataDir,inputFiles);
- if (inputFiles.size()==0) {
- throw new RuntimeException("No txt files in dataDir: "+dataDir.getAbsolutePath());
- }
- // date format: 30-MAR-1987 14:22:36.87
- dateFormat = new SimpleDateFormat[DATE_FORMATS.length];
- for (int i = 0; i < dateFormat.length; i++) {
- dateFormat[i] = new SimpleDateFormat(DATE_FORMATS[i],Locale.US);
- dateFormat[i].setLenient(true);
- }
- }
-
- private void openNextFile() throws NoMoreDataException, Exception {
- closeInputs();
- int retries = 0;
- while (true) {
- File f = null;
- synchronized (this) {
- if (nextFile >= inputFiles.size()) {
- // exhausted files, start a new round, unless forever set to false.
- if (!forever) {
- throw new NoMoreDataException();
- }
- nextFile = 0;
- iteration++;
- }
- f = (File) inputFiles.get(nextFile++);
- }
- System.out.println("opening: "+f+" length: "+f.length());
- try {
- zis = new GZIPInputStream(new BufferedInputStream(new FileInputStream(f)));
- reader = new BufferedReader(new InputStreamReader(zis));
- return;
- } catch (Exception e) {
- retries++;
- if (retries<20) {
- System.out.println("Skipping 'bad' file "+f.getAbsolutePath()+" #retries="+retries);
- continue;
- } else {
- throw new NoMoreDataException();
- }
- }
- }
- }
-
- private void closeInputs() {
- if (zis!=null) {
- try {
- zis.close();
- } catch (IOException e) {
- System.out.println("closeInputs(): Ingnoring error: "+e);
- e.printStackTrace();
- }
- zis = null;
- }
- if (reader!=null) {
- try {
- reader.close();
- } catch (IOException e) {
- System.out.println("closeInputs(): Ingnoring error: "+e);
- e.printStackTrace();
- }
- reader = null;
- }
- }
-
- // read until finding a line that starts with the specified prefix
- private StringBuffer read (String prefix, StringBuffer sb, boolean collectMatchLine, boolean collectAll) throws Exception {
- sb = (sb==null ? new StringBuffer() : sb);
- String sep = "";
- while (true) {
- String line = reader.readLine();
- if (line==null) {
- openNextFile();
- continue;
- }
- if (line.startsWith(prefix)) {
- if (collectMatchLine) {
- sb.append(sep+line);
- sep = newline;
- }
- break;
- }
- if (collectAll) {
- sb.append(sep+line);
- sep = newline;
- }
- }
- //System.out.println("read: "+sb);
- return sb;
- }
-
- protected DocData getNextDocData() throws NoMoreDataException, Exception {
- if (reader==null) {
- openNextFile();
- }
- // 1. skip until doc start
- read("<DOC>",null,false,false);
- // 2. name
- StringBuffer sb = read("<DOCNO>",null,true,false);
- String name = sb.substring("<DOCNO>".length());
- name = name.substring(0,name.indexOf("</DOCNO>"))+"_"+iteration;
- // 3. skip until doc header
- read("<DOCHDR>",null,false,false);
- // 4. date
- sb = read("Date: ",null,true,false);
- String dateStr = sb.substring("Date: ".length());
- // 5. skip until end of doc header
- read("</DOCHDR>",null,false,false);
- // 6. collect until end of doc
- sb = read("</DOC>",null,false,true);
- // this is the next document, so parse it
- Date date = parseDate(dateStr);
- HTMLParser p = getHtmlParser();
- DocData docData = p.parse(name, date, sb, dateFormat[0]);
- addBytes(sb.length()); // count char length of parsed html text (larger than the plain doc body text).
-
- return docData;
- }
-
- private Date parseDate(String dateStr) {
- Date date = null;
- for (int i=0; i<dateFormat.length; i++) {
- try {
- date = dateFormat[i].parse(dateStr.trim());
- return date;
- } catch (ParseException e) {
- }
- }
- // do not fail test just because a date could not be parsed
- System.out.println("ignoring date parse exception (assigning 'now') for: "+dateStr);
- date = new Date(); // now
- return date;
- }
-
-
- /*
- * (non-Javadoc)
- * @see DocMaker#resetIinputs()
- */
- public synchronized void resetInputs() {
- super.resetInputs();
- closeInputs();
- nextFile = 0;
- iteration = 0;
- }
-
- /*
- * (non-Javadoc)
- * @see DocMaker#numUniqueTexts()
- */
- public int numUniqueTexts() {
- return inputFiles.size();
- }
-
-}
+package org.apache.lucene.benchmark.byTask.feeds;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.Locale;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.lucene.benchmark.byTask.utils.Config;
+
+
+/**
+ * A DocMaker using the (compressed) Trec collection for its input.
+ */
+public class TrecDocMaker extends BasicDocMaker {
+
+ private static final String newline = System.getProperty("line.separator");
+
+ private DateFormat dateFormat [];
+ private File dataDir = null;
+ private ArrayList inputFiles = new ArrayList();
+ private int nextFile = 0;
+ private int iteration=0;
+ private BufferedReader reader;
+ private GZIPInputStream zis;
+
+ private static final String DATE_FORMATS [] = {
+ "EEE, dd MMM yyyy kk:mm:ss z", //Tue, 09 Dec 2003 22:39:08 GMT
+ "EEE MMM dd kk:mm:ss yyyy z", //Tue Dec 09 16:45:08 2003 EST
+ "EEE, dd-MMM-':'y kk:mm:ss z", //Tue, 09 Dec 2003 22:39:08 GMT
+ "EEE, dd-MMM-yyy kk:mm:ss z", //Tue, 09 Dec 2003 22:39:08 GMT
+ };
+
+ /* (non-Javadoc)
+ * @see SimpleDocMaker#setConfig(java.util.Properties)
+ */
+ public void setConfig(Config config) {
+ super.setConfig(config);
+ String d = config.get("docs.dir","trec");
+ dataDir = new File(new File("work"),d);
+ collectFiles(dataDir,inputFiles);
+ if (inputFiles.size()==0) {
+ throw new RuntimeException("No txt files in dataDir: "+dataDir.getAbsolutePath());
+ }
+ // date format: 30-MAR-1987 14:22:36.87
+ dateFormat = new SimpleDateFormat[DATE_FORMATS.length];
+ for (int i = 0; i < dateFormat.length; i++) {
+ dateFormat[i] = new SimpleDateFormat(DATE_FORMATS[i],Locale.US);
+ dateFormat[i].setLenient(true);
+ }
+ }
+
+ private void openNextFile() throws NoMoreDataException, Exception {
+ closeInputs();
+ int retries = 0;
+ while (true) {
+ File f = null;
+ synchronized (this) {
+ if (nextFile >= inputFiles.size()) {
+ // exhausted files, start a new round, unless forever set to false.
+ if (!forever) {
+ throw new NoMoreDataException();
+ }
+ nextFile = 0;
+ iteration++;
+ }
+ f = (File) inputFiles.get(nextFile++);
+ }
+ System.out.println("opening: "+f+" length: "+f.length());
+ try {
+ zis = new GZIPInputStream(new BufferedInputStream(new FileInputStream(f)));
+ reader = new BufferedReader(new InputStreamReader(zis));
+ return;
+ } catch (Exception e) {
+ retries++;
+ if (retries<20) {
+ System.out.println("Skipping 'bad' file "+f.getAbsolutePath()+" #retries="+retries);
+ continue;
+ } else {
+ throw new NoMoreDataException();
+ }
+ }
+ }
+ }
+
+ private void closeInputs() {
+ if (zis!=null) {
+ try {
+ zis.close();
+ } catch (IOException e) {
+ System.out.println("closeInputs(): Ingnoring error: "+e);
+ e.printStackTrace();
+ }
+ zis = null;
+ }
+ if (reader!=null) {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ System.out.println("closeInputs(): Ingnoring error: "+e);
+ e.printStackTrace();
+ }
+ reader = null;
+ }
+ }
+
+ // read until finding a line that starts with the specified prefix
+ private StringBuffer read (String prefix, StringBuffer sb, boolean collectMatchLine, boolean collectAll) throws Exception {
+ sb = (sb==null ? new StringBuffer() : sb);
+ String sep = "";
+ while (true) {
+ String line = reader.readLine();
+ if (line==null) {
+ openNextFile();
+ continue;
+ }
+ if (line.startsWith(prefix)) {
+ if (collectMatchLine) {
+ sb.append(sep+line);
+ sep = newline;
+ }
+ break;
+ }
+ if (collectAll) {
+ sb.append(sep+line);
+ sep = newline;
+ }
+ }
+ //System.out.println("read: "+sb);
+ return sb;
+ }
+
+ protected DocData getNextDocData() throws NoMoreDataException, Exception {
+ if (reader==null) {
+ openNextFile();
+ }
+ // 1. skip until doc start
+ read("<DOC>",null,false,false);
+ // 2. name
+ StringBuffer sb = read("<DOCNO>",null,true,false);
+ String name = sb.substring("<DOCNO>".length());
+ name = name.substring(0,name.indexOf("</DOCNO>"))+"_"+iteration;
+ // 3. skip until doc header
+ read("<DOCHDR>",null,false,false);
+ // 4. date
+ sb = read("Date: ",null,true,false);
+ String dateStr = sb.substring("Date: ".length());
+ // 5. skip until end of doc header
+ read("</DOCHDR>",null,false,false);
+ // 6. collect until end of doc
+ sb = read("</DOC>",null,false,true);
+ // this is the next document, so parse it
+ Date date = parseDate(dateStr);
+ HTMLParser p = getHtmlParser();
+ DocData docData = p.parse(name, date, sb, dateFormat[0]);
+ addBytes(sb.length()); // count char length of parsed html text (larger than the plain doc body text).
+
+ return docData;
+ }
+
+ private Date parseDate(String dateStr) {
+ Date date = null;
+ for (int i=0; i<dateFormat.length; i++) {
+ try {
+ date = dateFormat[i].parse(dateStr.trim());
+ return date;
+ } catch (ParseException e) {
+ }
+ }
+ // do not fail test just because a date could not be parsed
+ System.out.println("ignoring date parse exception (assigning 'now') for: "+dateStr);
+ date = new Date(); // now
+ return date;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ * @see DocMaker#resetIinputs()
+ */
+ public synchronized void resetInputs() {
+ super.resetInputs();
+ closeInputs();
+ nextFile = 0;
+ iteration = 0;
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see DocMaker#numUniqueTexts()
+ */
+ public int numUniqueTexts() {
+ return inputFiles.size();
+ }
+
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocMaker.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/programmatic/Sample.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/programmatic/Sample.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/programmatic/Sample.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/programmatic/Sample.java Mon Apr 2 16:37:14 2007
@@ -1,101 +1,101 @@
-package org.apache.lucene.benchmark.byTask.programmatic;
-
-/**
- * Copyright 2005 The Apache Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.util.Properties;
-
-import org.apache.lucene.benchmark.byTask.PerfRunData;
-import org.apache.lucene.benchmark.byTask.tasks.AddDocTask;
-import org.apache.lucene.benchmark.byTask.tasks.CloseIndexTask;
-import org.apache.lucene.benchmark.byTask.tasks.CreateIndexTask;
-import org.apache.lucene.benchmark.byTask.tasks.RepSumByNameTask;
-import org.apache.lucene.benchmark.byTask.tasks.TaskSequence;
-import org.apache.lucene.benchmark.byTask.utils.Config;
-
-/**
- * Sample performance test written programatically - no algorithm file is needed here.
- */
-public class Sample {
-
- /**
- * @param args
- * @throws Exception
- * @throws IOException
- */
- public static void main(String[] args) throws Exception {
- Properties p = initProps();
- Config conf = new Config(p);
- PerfRunData runData = new PerfRunData(conf);
-
- // 1. top sequence
- TaskSequence top = new TaskSequence(runData,null,null,false); // top level, not parralel
-
- // 2. task to create the index
- CreateIndexTask create = new CreateIndexTask(runData);
- top.addTask(create);
-
- // 3. task seq to add 500 docs (order matters - top to bottom - add seq to top, only then add to seq)
- TaskSequence seq1 = new TaskSequence(runData,"AddDocs",top,false);
- seq1.setRepetitions(500);
- seq1.setNoChildReport();
- top.addTask(seq1);
-
- // 4. task to add the doc
- AddDocTask addDoc = new AddDocTask(runData);
- //addDoc.setParams("1200"); // doc size limit if supported
- seq1.addTask(addDoc); // order matters 9see comment above)
-
- // 5. task to close the index
- CloseIndexTask close = new CloseIndexTask(runData);
- top.addTask(close);
-
- // task to report
- RepSumByNameTask rep = new RepSumByNameTask(runData);
- top.addTask(rep);
-
- // print algorithm
- System.out.println(top.toString());
-
- // execute
- top.doLogic();
- }
-
- // Sample programmatic settings. Could also read from file.
- private static Properties initProps() {
- Properties p = new Properties();
- p.setProperty ( "task.max.depth.log" , "3" );
- p.setProperty ( "max.buffered" , "buf:10:10:100:100:10:10:100:100" );
- p.setProperty ( "doc.maker" , "org.apache.lucene.benchmark.byTask.feeds.ReutersDocMaker" );
- p.setProperty ( "doc.add.log.step" , "2000" );
- p.setProperty ( "doc.delete.log.step" , "2000" );
- p.setProperty ( "doc.delete.step" , "8" );
- p.setProperty ( "analyzer" , "org.apache.lucene.analysis.standard.StandardAnalyzer" );
- p.setProperty ( "doc.term.vector" , "false" );
- p.setProperty ( "directory" , "FSDirectory" );
- p.setProperty ( "query.maker" , "org.apache.lucene.benchmark.byTask.feeds.ReutersQueryMaker" );
- p.setProperty ( "doc.stored" , "true" );
- p.setProperty ( "docs.dir" , "reuters-out" );
- p.setProperty ( "compound" , "cmpnd:true:true:true:true:false:false:false:false" );
- p.setProperty ( "doc.tokenized" , "true" );
- p.setProperty ( "merge.factor" , "mrg:10:100:10:100:10:100:10:100" );
- return p;
- }
-
-
-
-}
+package org.apache.lucene.benchmark.byTask.programmatic;
+
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+import org.apache.lucene.benchmark.byTask.tasks.AddDocTask;
+import org.apache.lucene.benchmark.byTask.tasks.CloseIndexTask;
+import org.apache.lucene.benchmark.byTask.tasks.CreateIndexTask;
+import org.apache.lucene.benchmark.byTask.tasks.RepSumByNameTask;
+import org.apache.lucene.benchmark.byTask.tasks.TaskSequence;
+import org.apache.lucene.benchmark.byTask.utils.Config;
+
+/**
+ * Sample performance test written programatically - no algorithm file is needed here.
+ */
+public class Sample {
+
+ /**
+ * @param args
+ * @throws Exception
+ * @throws IOException
+ */
+ public static void main(String[] args) throws Exception {
+ Properties p = initProps();
+ Config conf = new Config(p);
+ PerfRunData runData = new PerfRunData(conf);
+
+ // 1. top sequence
+ TaskSequence top = new TaskSequence(runData,null,null,false); // top level, not parralel
+
+ // 2. task to create the index
+ CreateIndexTask create = new CreateIndexTask(runData);
+ top.addTask(create);
+
+ // 3. task seq to add 500 docs (order matters - top to bottom - add seq to top, only then add to seq)
+ TaskSequence seq1 = new TaskSequence(runData,"AddDocs",top,false);
+ seq1.setRepetitions(500);
+ seq1.setNoChildReport();
+ top.addTask(seq1);
+
+ // 4. task to add the doc
+ AddDocTask addDoc = new AddDocTask(runData);
+ //addDoc.setParams("1200"); // doc size limit if supported
+ seq1.addTask(addDoc); // order matters 9see comment above)
+
+ // 5. task to close the index
+ CloseIndexTask close = new CloseIndexTask(runData);
+ top.addTask(close);
+
+ // task to report
+ RepSumByNameTask rep = new RepSumByNameTask(runData);
+ top.addTask(rep);
+
+ // print algorithm
+ System.out.println(top.toString());
+
+ // execute
+ top.doLogic();
+ }
+
+ // Sample programmatic settings. Could also read from file.
+ private static Properties initProps() {
+ Properties p = new Properties();
+ p.setProperty ( "task.max.depth.log" , "3" );
+ p.setProperty ( "max.buffered" , "buf:10:10:100:100:10:10:100:100" );
+ p.setProperty ( "doc.maker" , "org.apache.lucene.benchmark.byTask.feeds.ReutersDocMaker" );
+ p.setProperty ( "doc.add.log.step" , "2000" );
+ p.setProperty ( "doc.delete.log.step" , "2000" );
+ p.setProperty ( "doc.delete.step" , "8" );
+ p.setProperty ( "analyzer" , "org.apache.lucene.analysis.standard.StandardAnalyzer" );
+ p.setProperty ( "doc.term.vector" , "false" );
+ p.setProperty ( "directory" , "FSDirectory" );
+ p.setProperty ( "query.maker" , "org.apache.lucene.benchmark.byTask.feeds.ReutersQueryMaker" );
+ p.setProperty ( "doc.stored" , "true" );
+ p.setProperty ( "docs.dir" , "reuters-out" );
+ p.setProperty ( "compound" , "cmpnd:true:true:true:true:false:false:false:false" );
+ p.setProperty ( "doc.tokenized" , "true" );
+ p.setProperty ( "merge.factor" , "mrg:10:100:10:100:10:100:10:100" );
+ return p;
+ }
+
+
+
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/programmatic/Sample.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/Points.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/Points.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/Points.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/Points.java Mon Apr 2 16:37:14 2007
@@ -1,91 +1,91 @@
-package org.apache.lucene.benchmark.byTask.stats;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.util.ArrayList;
+package org.apache.lucene.benchmark.byTask.stats;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
import java.util.List;
-
-import org.apache.lucene.benchmark.byTask.tasks.PerfTask;
-import org.apache.lucene.benchmark.byTask.utils.Config;
-
-
-/**
- * Test run data points collected as the test proceeds.
- */
-public class Points {
-
- private Config config;
-
- // stat points ordered by their start time.
- // for now we collect points as TaskStats objects.
- // later might optimize to collect only native data.
- private ArrayList points = new ArrayList();
-
- private int nextTaskRunNum = 0;
-
- /**
- * Create a Points statistics object.
- */
- public Points (Config config) {
- this.config = config;
- }
-
- /**
+
+import org.apache.lucene.benchmark.byTask.tasks.PerfTask;
+import org.apache.lucene.benchmark.byTask.utils.Config;
+
+
+/**
+ * Test run data points collected as the test proceeds.
+ */
+public class Points {
+
+ private Config config;
+
+ // stat points ordered by their start time.
+ // for now we collect points as TaskStats objects.
+ // later might optimize to collect only native data.
+ private ArrayList points = new ArrayList();
+
+ private int nextTaskRunNum = 0;
+
+ /**
+ * Create a Points statistics object.
+ */
+ public Points (Config config) {
+ this.config = config;
+ }
+
+ /**
* Return the current task stats.
* the actual task stats are returned, so caller should not modify this task stats.
* @return current {@link TaskStats}.
- */
+ */
public List taskStats () {
return points;
- }
-
- /**
- * Mark that a task is starting.
- * Create a task stats for it and store it as a point.
- * @param task the starting task.
- * @return the new task stats created for the starting task.
- */
- public synchronized TaskStats markTaskStart (PerfTask task, int round) {
- TaskStats stats = new TaskStats(task, nextTaskRunNum(), round);
- points.add(stats);
- return stats;
- }
-
- // return next task num
- private synchronized int nextTaskRunNum() {
- return nextTaskRunNum++;
- }
-
- /**
- * mark the end of a task
- */
- public synchronized void markTaskEnd (TaskStats stats, int count) {
- int numParallelTasks = nextTaskRunNum - 1 - stats.getTaskRunNum();
- // note: if the stats were cleared, might be that this stats object is
- // no longer in points, but this is just ok.
- stats.markEnd(numParallelTasks, count);
- }
-
- /**
- * Clear all data, prepare for more tests.
- */
- public void clearData() {
- points.clear();
- }
-
-}
+ }
+
+ /**
+ * Mark that a task is starting.
+ * Create a task stats for it and store it as a point.
+ * @param task the starting task.
+ * @return the new task stats created for the starting task.
+ */
+ public synchronized TaskStats markTaskStart (PerfTask task, int round) {
+ TaskStats stats = new TaskStats(task, nextTaskRunNum(), round);
+ points.add(stats);
+ return stats;
+ }
+
+ // return next task num
+ private synchronized int nextTaskRunNum() {
+ return nextTaskRunNum++;
+ }
+
+ /**
+ * mark the end of a task
+ */
+ public synchronized void markTaskEnd (TaskStats stats, int count) {
+ int numParallelTasks = nextTaskRunNum - 1 - stats.getTaskRunNum();
+ // note: if the stats were cleared, might be that this stats object is
+ // no longer in points, but this is just ok.
+ stats.markEnd(numParallelTasks, count);
+ }
+
+ /**
+ * Clear all data, prepare for more tests.
+ */
+ public void clearData() {
+ points.clear();
+ }
+
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/Points.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/Report.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/Report.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/Report.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/Report.java Mon Apr 2 16:37:14 2007
@@ -1,64 +1,64 @@
-package org.apache.lucene.benchmark.byTask.stats;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Textual report of current statistics.
- */
-public class Report {
-
- private String text;
- private int size;
- private int outOf;
- private int reported;
-
+package org.apache.lucene.benchmark.byTask.stats;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Textual report of current statistics.
+ */
+public class Report {
+
+ private String text;
+ private int size;
+ private int outOf;
+ private int reported;
+
public Report (String text, int size, int reported, int outOf) {
- this.text = text;
- this.size = size;
- this.reported = reported;
- this.outOf = outOf;
- }
-
- /**
- * Returns total number of stats points when this report was created.
- */
- public int getOutOf() {
- return outOf;
- }
-
- /**
- * Returns number of lines in the reoprt.
- */
- public int getSize() {
- return size;
- }
-
- /**
- * Returns the report text.
- */
- public String getText() {
- return text;
- }
-
- /**
- * Returns number of stats points represented in this report.
- */
- public int getReported() {
- return reported;
- }
-}
+ this.text = text;
+ this.size = size;
+ this.reported = reported;
+ this.outOf = outOf;
+ }
+
+ /**
+ * Returns total number of stats points when this report was created.
+ */
+ public int getOutOf() {
+ return outOf;
+ }
+
+ /**
+ * Returns number of lines in the reoprt.
+ */
+ public int getSize() {
+ return size;
+ }
+
+ /**
+ * Returns the report text.
+ */
+ public String getText() {
+ return text;
+ }
+
+ /**
+ * Returns number of stats points represented in this report.
+ */
+ public int getReported() {
+ return reported;
+ }
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/Report.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/TaskStats.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/TaskStats.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/TaskStats.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/TaskStats.java Mon Apr 2 16:37:14 2007
@@ -1,192 +1,192 @@
-package org.apache.lucene.benchmark.byTask.stats;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.benchmark.byTask.tasks.PerfTask;
-
-/**
- * Statistics for a task run.
- * <br>The same task can run more than once, but, if that task records statistics,
- * each run would create its own TaskStats.
- */
-public class TaskStats implements Cloneable {
-
- /** task for which data was collected */
- private PerfTask task;
-
- /** round in which task run started */
- private int round;
-
- /** task start time */
- private long start;
-
- /** task elapsed time. elapsed >= 0 indicates run completion! */
- private long elapsed = -1;
-
- /** max tot mem during task */
- private long maxTotMem;
-
- /** max used mem during task */
- private long maxUsedMem;
-
- /** serial run number of this task run in the perf run */
- private int taskRunNum;
-
- /** number of other tasks that started to run while this task was still running */
- private int numParallelTasks;
-
- /** number of work items done by this task.
- * For indexing that can be number of docs added.
- * For warming that can be number of scanned items, etc.
- * For repeating tasks, this is a sum over repetitions.
- */
- private int count;
-
- /** Number of similar tasks aggregated into this record.
- * Used when summing up on few runs/instances of similar tasks.
- */
- private int numRuns = 1;
-
- /**
- * Create a run data for a task that is starting now.
- * To be called from Points.
- */
- TaskStats (PerfTask task, int taskRunNum, int round) {
- this.task = task;
- this.taskRunNum = taskRunNum;
- this.round = round;
- maxTotMem = Runtime.getRuntime().totalMemory();
- maxUsedMem = maxTotMem - Runtime.getRuntime().freeMemory();
- start = System.currentTimeMillis();
- }
-
- /**
- * mark the end of a task
- */
- void markEnd (int numParallelTasks, int count) {
- elapsed = System.currentTimeMillis() - start;
- long totMem = Runtime.getRuntime().totalMemory();
- if (totMem > maxTotMem) {
- maxTotMem = totMem;
- }
- long usedMem = totMem - Runtime.getRuntime().freeMemory();
- if (usedMem > maxUsedMem) {
- maxUsedMem = usedMem;
- }
- this.numParallelTasks = numParallelTasks;
- this.count = count;
- }
-
- /**
- * @return the taskRunNum.
- */
- public int getTaskRunNum() {
- return taskRunNum;
- }
-
- /* (non-Javadoc)
- * @see java.lang.Object#toString()
- */
- public String toString() {
- StringBuffer res = new StringBuffer(task.getName());
- res.append(" ");
- res.append(count);
- res.append(" ");
- res.append(elapsed);
- return res.toString();
- }
-
- /**
- * @return Returns the count.
- */
- public int getCount() {
- return count;
- }
-
- /**
- * @return elapsed time.
- */
- public long getElapsed() {
- return elapsed;
- }
-
- /**
- * @return Returns the maxTotMem.
- */
- public long getMaxTotMem() {
- return maxTotMem;
- }
-
- /**
- * @return Returns the maxUsedMem.
- */
- public long getMaxUsedMem() {
- return maxUsedMem;
- }
-
- /**
- * @return Returns the numParallelTasks.
- */
- public int getNumParallelTasks() {
- return numParallelTasks;
- }
-
- /**
- * @return Returns the task.
- */
- public PerfTask getTask() {
- return task;
- }
-
- /**
- * @return Returns the numRuns.
- */
- public int getNumRuns() {
- return numRuns;
- }
-
- /**
- * Add data from another stat, for aggregation
- * @param stat2 the added stat data.
- */
- public void add(TaskStats stat2) {
- numRuns += stat2.getNumRuns();
- elapsed += stat2.getElapsed();
- maxTotMem += stat2.getMaxTotMem();
- maxUsedMem += stat2.getMaxUsedMem();
- count += stat2.getCount();
- if (round != stat2.round) {
- round = -1; // no meaning if agregating tasks of different ruond.
- }
- }
-
- /* (non-Javadoc)
- * @see java.lang.Object#clone()
- */
+package org.apache.lucene.benchmark.byTask.stats;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.benchmark.byTask.tasks.PerfTask;
+
+/**
+ * Statistics for a task run.
+ * <br>The same task can run more than once, but, if that task records statistics,
+ * each run would create its own TaskStats.
+ */
+public class TaskStats implements Cloneable {
+
+ /** task for which data was collected */
+ private PerfTask task;
+
+ /** round in which task run started */
+ private int round;
+
+ /** task start time */
+ private long start;
+
+ /** task elapsed time. elapsed >= 0 indicates run completion! */
+ private long elapsed = -1;
+
+ /** max tot mem during task */
+ private long maxTotMem;
+
+ /** max used mem during task */
+ private long maxUsedMem;
+
+ /** serial run number of this task run in the perf run */
+ private int taskRunNum;
+
+ /** number of other tasks that started to run while this task was still running */
+ private int numParallelTasks;
+
+ /** number of work items done by this task.
+ * For indexing that can be number of docs added.
+ * For warming that can be number of scanned items, etc.
+ * For repeating tasks, this is a sum over repetitions.
+ */
+ private int count;
+
+ /** Number of similar tasks aggregated into this record.
+ * Used when summing up on few runs/instances of similar tasks.
+ */
+ private int numRuns = 1;
+
+ /**
+ * Create a run data for a task that is starting now.
+ * To be called from Points.
+ */
+ TaskStats (PerfTask task, int taskRunNum, int round) {
+ this.task = task;
+ this.taskRunNum = taskRunNum;
+ this.round = round;
+ maxTotMem = Runtime.getRuntime().totalMemory();
+ maxUsedMem = maxTotMem - Runtime.getRuntime().freeMemory();
+ start = System.currentTimeMillis();
+ }
+
+ /**
+ * mark the end of a task
+ */
+ void markEnd (int numParallelTasks, int count) {
+ elapsed = System.currentTimeMillis() - start;
+ long totMem = Runtime.getRuntime().totalMemory();
+ if (totMem > maxTotMem) {
+ maxTotMem = totMem;
+ }
+ long usedMem = totMem - Runtime.getRuntime().freeMemory();
+ if (usedMem > maxUsedMem) {
+ maxUsedMem = usedMem;
+ }
+ this.numParallelTasks = numParallelTasks;
+ this.count = count;
+ }
+
+ /**
+ * @return the taskRunNum.
+ */
+ public int getTaskRunNum() {
+ return taskRunNum;
+ }
+
+ /* (non-Javadoc)
+ * @see java.lang.Object#toString()
+ */
+ public String toString() {
+ StringBuffer res = new StringBuffer(task.getName());
+ res.append(" ");
+ res.append(count);
+ res.append(" ");
+ res.append(elapsed);
+ return res.toString();
+ }
+
+ /**
+ * @return Returns the count.
+ */
+ public int getCount() {
+ return count;
+ }
+
+ /**
+ * @return elapsed time.
+ */
+ public long getElapsed() {
+ return elapsed;
+ }
+
+ /**
+ * @return Returns the maxTotMem.
+ */
+ public long getMaxTotMem() {
+ return maxTotMem;
+ }
+
+ /**
+ * @return Returns the maxUsedMem.
+ */
+ public long getMaxUsedMem() {
+ return maxUsedMem;
+ }
+
+ /**
+ * @return Returns the numParallelTasks.
+ */
+ public int getNumParallelTasks() {
+ return numParallelTasks;
+ }
+
+ /**
+ * @return Returns the task.
+ */
+ public PerfTask getTask() {
+ return task;
+ }
+
+ /**
+ * @return Returns the numRuns.
+ */
+ public int getNumRuns() {
+ return numRuns;
+ }
+
+ /**
+ * Add data from another stat, for aggregation
+ * @param stat2 the added stat data.
+ */
+ public void add(TaskStats stat2) {
+ numRuns += stat2.getNumRuns();
+ elapsed += stat2.getElapsed();
+ maxTotMem += stat2.getMaxTotMem();
+ maxUsedMem += stat2.getMaxUsedMem();
+ count += stat2.getCount();
+ if (round != stat2.round) {
+ round = -1; // no meaning if agregating tasks of different ruond.
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see java.lang.Object#clone()
+ */
public Object clone() throws CloneNotSupportedException {
- return super.clone();
- }
-
- /**
- * @return the round number.
- */
+ return super.clone();
+ }
+
+ /**
+ * @return the round number.
+ */
public int getRound() {
- return round;
- }
-
-}
+ return round;
+ }
+
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/stats/TaskStats.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java Mon Apr 2 16:37:14 2007
@@ -1,98 +1,98 @@
-package org.apache.lucene.benchmark.byTask.tasks;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.benchmark.byTask.PerfRunData;
-import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
-import org.apache.lucene.document.Document;
-
-
-/**
+package org.apache.lucene.benchmark.byTask.tasks;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
+import org.apache.lucene.document.Document;
+
+
+/**
* Add a document, optionally with of a certain size.
* <br>Other side effects: none.
* <br>Relevant properties: <code>doc.add.log.step</code>.
* <br>Takes optional param: document size.
- */
-public class AddDocTask extends PerfTask {
-
+ */
+public class AddDocTask extends PerfTask {
+
/**
* Default value for property <code>doc.add.log.step<code> - indicating how often
* an "added N docs" message should be logged.
*/
public static final int DEFAULT_ADD_DOC_LOG_STEP = 500;
- public AddDocTask(PerfRunData runData) {
- super(runData);
- }
-
- private static int logStep = -1;
- private int docSize = 0;
-
- // volatile data passed between setup(), doLogic(), tearDown().
- private Document doc = null;
-
- /*
- * (non-Javadoc)
- * @see PerfTask#setup()
- */
- public void setup() throws Exception {
- super.setup();
- DocMaker docMaker = getRunData().getDocMaker();
- if (docSize > 0) {
- doc = docMaker.makeDocument(docSize);
- } else {
- doc = docMaker.makeDocument();
- }
- }
-
- /* (non-Javadoc)
- * @see PerfTask#tearDown()
- */
- public void tearDown() throws Exception {
- DocMaker docMaker = getRunData().getDocMaker();
- log(docMaker.getCount());
- doc = null;
- super.tearDown();
- }
-
- public int doLogic() throws Exception {
- getRunData().getIndexWriter().addDocument(doc);
- return 1;
- }
-
- private void log (int count) {
- if (logStep<0) {
- // avoid sync although race possible here
+ public AddDocTask(PerfRunData runData) {
+ super(runData);
+ }
+
+ private static int logStep = -1;
+ private int docSize = 0;
+
+ // volatile data passed between setup(), doLogic(), tearDown().
+ private Document doc = null;
+
+ /*
+ * (non-Javadoc)
+ * @see PerfTask#setup()
+ */
+ public void setup() throws Exception {
+ super.setup();
+ DocMaker docMaker = getRunData().getDocMaker();
+ if (docSize > 0) {
+ doc = docMaker.makeDocument(docSize);
+ } else {
+ doc = docMaker.makeDocument();
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see PerfTask#tearDown()
+ */
+ public void tearDown() throws Exception {
+ DocMaker docMaker = getRunData().getDocMaker();
+ log(docMaker.getCount());
+ doc = null;
+ super.tearDown();
+ }
+
+ public int doLogic() throws Exception {
+ getRunData().getIndexWriter().addDocument(doc);
+ return 1;
+ }
+
+ private void log (int count) {
+ if (logStep<0) {
+ // avoid sync although race possible here
logStep = getRunData().getConfig().get("doc.add.log.step",DEFAULT_ADD_DOC_LOG_STEP);
- }
- if (logStep>0 && (count%logStep)==0) {
+ }
+ if (logStep>0 && (count%logStep)==0) {
System.out.println("--> processed (add) "+count+" docs");
- }
- }
-
- /**
- * Set the params (docSize only)
- * @param params docSize, or 0 for no limit.
- */
- public void setParams(String params) {
- super.setParams(params);
- docSize = (int) Float.parseFloat(params);
- }
+ }
+ }
+
+ /**
+ * Set the params (docSize only)
+ * @param params docSize, or 0 for no limit.
+ */
+ public void setParams(String params) {
+ super.setParams(params);
+ docSize = (int) Float.parseFloat(params);
+ }
/* (non-Javadoc)
* @see org.apache.lucene.benchmark.byTask.tasks.PerfTask#supportsParams()
@@ -101,4 +101,4 @@
return true;
}
-}
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddDocTask.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ClearStatsTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ClearStatsTask.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ClearStatsTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ClearStatsTask.java Mon Apr 2 16:37:14 2007
@@ -1,44 +1,44 @@
-package org.apache.lucene.benchmark.byTask.tasks;
-
-import org.apache.lucene.benchmark.byTask.PerfRunData;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Clear statistics data.
+package org.apache.lucene.benchmark.byTask.tasks;
+
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Clear statistics data.
* <br>Other side effects: None.
- */
-public class ClearStatsTask extends PerfTask {
-
- public ClearStatsTask(PerfRunData runData) {
- super(runData);
- }
-
- public int doLogic() throws Exception {
- getRunData().getPoints().clearData();
- return 0;
- }
-
- /* (non-Javadoc)
- * @see PerfTask#shouldNotRecordStats()
- */
- protected boolean shouldNotRecordStats() {
- return true;
- }
-
-}
+ */
+public class ClearStatsTask extends PerfTask {
+
+ public ClearStatsTask(PerfRunData runData) {
+ super(runData);
+ }
+
+ public int doLogic() throws Exception {
+ getRunData().getPoints().clearData();
+ return 0;
+ }
+
+ /* (non-Javadoc)
+ * @see PerfTask#shouldNotRecordStats()
+ */
+ protected boolean shouldNotRecordStats() {
+ return true;
+ }
+
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ClearStatsTask.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseIndexTask.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseIndexTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseIndexTask.java Mon Apr 2 16:37:14 2007
@@ -1,44 +1,44 @@
-package org.apache.lucene.benchmark.byTask.tasks;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-
-import org.apache.lucene.benchmark.byTask.PerfRunData;
-import org.apache.lucene.index.IndexWriter;
-
-/**
- * Close index writer.
+package org.apache.lucene.benchmark.byTask.tasks;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+import org.apache.lucene.index.IndexWriter;
+
+/**
+ * Close index writer.
* <br>Other side effects: index writer object in perfRunData is nullified.
- */
-public class CloseIndexTask extends PerfTask {
-
- public CloseIndexTask(PerfRunData runData) {
- super(runData);
- }
-
- public int doLogic() throws IOException {
- IndexWriter iw = getRunData().getIndexWriter();
- if (iw!=null) {
- iw.close();
- }
- getRunData().setIndexWriter(null);
- return 1;
- }
-
-}
+ */
+public class CloseIndexTask extends PerfTask {
+
+ public CloseIndexTask(PerfRunData runData) {
+ super(runData);
+ }
+
+ public int doLogic() throws IOException {
+ IndexWriter iw = getRunData().getIndexWriter();
+ if (iw!=null) {
+ iw.close();
+ }
+ getRunData().setIndexWriter(null);
+ return 1;
+ }
+
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseIndexTask.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseReaderTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseReaderTask.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseReaderTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseReaderTask.java Mon Apr 2 16:37:14 2007
@@ -1,45 +1,45 @@
-package org.apache.lucene.benchmark.byTask.tasks;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-
-import org.apache.lucene.benchmark.byTask.PerfRunData;
-import org.apache.lucene.index.IndexReader;
-
-/**
- * Close index reader.
+package org.apache.lucene.benchmark.byTask.tasks;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+import org.apache.lucene.index.IndexReader;
+
+/**
+ * Close index reader.
* <br>Other side effects: index reader in perfRunData is nullified.
* <br>This would cause read related tasks to reopen their own reader.
- */
-public class CloseReaderTask extends PerfTask {
-
- public CloseReaderTask(PerfRunData runData) {
- super(runData);
- }
-
- public int doLogic() throws IOException {
- IndexReader reader= getRunData().getIndexReader();
- if (reader!=null) {
- reader.close();
- }
- getRunData().setIndexReader(null);
- return 1;
- }
-
-}
+ */
+public class CloseReaderTask extends PerfTask {
+
+ public CloseReaderTask(PerfRunData runData) {
+ super(runData);
+ }
+
+ public int doLogic() throws IOException {
+ IndexReader reader= getRunData().getIndexReader();
+ if (reader!=null) {
+ reader.close();
+ }
+ getRunData().setIndexReader(null);
+ return 1;
+ }
+
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CloseReaderTask.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java Mon Apr 2 16:37:14 2007
@@ -1,60 +1,60 @@
-package org.apache.lucene.benchmark.byTask.tasks;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.benchmark.byTask.PerfRunData;
-import org.apache.lucene.benchmark.byTask.utils.Config;
-
-
-/**
- * Create an index.
+package org.apache.lucene.benchmark.byTask.tasks;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+import org.apache.lucene.benchmark.byTask.utils.Config;
+
+
+/**
+ * Create an index.
* <br>Other side effects: index writer object in perfRunData is set.
* <br>Relevant properties: <code>merge.factor , max.buffered</code>.
- */
-public class CreateIndexTask extends PerfTask {
-
- public CreateIndexTask(PerfRunData runData) {
- super(runData);
- }
-
- public int doLogic() throws IOException {
- Directory dir = getRunData().getDirectory();
- Analyzer analyzer = getRunData().getAnalyzer();
-
- IndexWriter iw = new IndexWriter(dir, analyzer, true);
-
- Config config = getRunData().getConfig();
-
- boolean cmpnd = config.get("compound",true);
+ */
+public class CreateIndexTask extends PerfTask {
+
+ public CreateIndexTask(PerfRunData runData) {
+ super(runData);
+ }
+
+ public int doLogic() throws IOException {
+ Directory dir = getRunData().getDirectory();
+ Analyzer analyzer = getRunData().getAnalyzer();
+
+ IndexWriter iw = new IndexWriter(dir, analyzer, true);
+
+ Config config = getRunData().getConfig();
+
+ boolean cmpnd = config.get("compound",true);
int mrgf = config.get("merge.factor",OpenIndexTask.DEFAULT_MERGE_PFACTOR);
int mxbf = config.get("max.buffered",OpenIndexTask.DEFAULT_MAX_BUFFERED);
-
- iw.setUseCompoundFile(cmpnd);
- iw.setMergeFactor(mrgf);
- iw.setMaxBufferedDocs(mxbf);
-
- getRunData().setIndexWriter(iw);
- return 1;
- }
-
-}
+
+ iw.setUseCompoundFile(cmpnd);
+ iw.setMergeFactor(mrgf);
+ iw.setMaxBufferedDocs(mxbf);
+
+ getRunData().setIndexWriter(iw);
+ return 1;
+ }
+
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteDocTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteDocTask.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteDocTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteDocTask.java Mon Apr 2 16:37:14 2007
@@ -1,33 +1,33 @@
-package org.apache.lucene.benchmark.byTask.tasks;
-
-import org.apache.lucene.benchmark.byTask.PerfRunData;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Delete a document by docid.
+package org.apache.lucene.benchmark.byTask.tasks;
+
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Delete a document by docid.
* <br>Other side effects: none.
* <br>Relevant properties: <code>doc.delete.log.step , doc.delete.step</code>.
* <br>If no docid param is supplied, deletes doc with <code>id = last-deleted-doc + doc.delete.step</code>.
* <br>Takes optional param: document id.
- */
-public class DeleteDocTask extends PerfTask {
-
+ */
+public class DeleteDocTask extends PerfTask {
+
/**
* Gap between ids of deleted docs, applies when no docid param is provided.
*/
@@ -39,63 +39,63 @@
*/
public static final int DEFAULT_DELETE_DOC_LOG_STEP = 500;
- public DeleteDocTask(PerfRunData runData) {
- super(runData);
- }
-
- private static int logStep = -1;
- private static int deleteStep = -1;
- private static int numDeleted = 0;
- private static int lastDeleted = -1;
-
- private int docid = -1;
- private boolean byStep = true;
-
- public int doLogic() throws Exception {
- getRunData().getIndexReader().deleteDocument(docid);
- lastDeleted = docid;
- return 1; // one work item done here
- }
-
- /* (non-Javadoc)
- * @see org.apache.lucene.benchmark.byTask.tasks.PerfTask#setup()
- */
- public void setup() throws Exception {
- super.setup();
- // one time static initializations
- if (logStep<0) {
+ public DeleteDocTask(PerfRunData runData) {
+ super(runData);
+ }
+
+ private static int logStep = -1;
+ private static int deleteStep = -1;
+ private static int numDeleted = 0;
+ private static int lastDeleted = -1;
+
+ private int docid = -1;
+ private boolean byStep = true;
+
+ public int doLogic() throws Exception {
+ getRunData().getIndexReader().deleteDocument(docid);
+ lastDeleted = docid;
+ return 1; // one work item done here
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.benchmark.byTask.tasks.PerfTask#setup()
+ */
+ public void setup() throws Exception {
+ super.setup();
+ // one time static initializations
+ if (logStep<0) {
logStep = getRunData().getConfig().get("doc.delete.log.step",DEFAULT_DELETE_DOC_LOG_STEP);
- }
- if (deleteStep<0) {
+ }
+ if (deleteStep<0) {
deleteStep = getRunData().getConfig().get("doc.delete.step",DEFAULT_DOC_DELETE_STEP);
- }
- // set the docid to be deleted
- docid = (byStep ? lastDeleted + deleteStep : docid);
- }
-
- /* (non-Javadoc)
- * @see PerfTask#tearDown()
- */
- public void tearDown() throws Exception {
- log(++numDeleted);
- super.tearDown();
- }
-
- private void log (int count) {
- if (logStep>0 && (count%logStep)==0) {
+ }
+ // set the docid to be deleted
+ docid = (byStep ? lastDeleted + deleteStep : docid);
+ }
+
+ /* (non-Javadoc)
+ * @see PerfTask#tearDown()
+ */
+ public void tearDown() throws Exception {
+ log(++numDeleted);
+ super.tearDown();
+ }
+
+ private void log (int count) {
+ if (logStep>0 && (count%logStep)==0) {
System.out.println("--> processed (delete) "+count+" docs, last deleted: "+lastDeleted);
- }
- }
-
- /**
- * Set the params (docid only)
- * @param params docid to delete, or -1 for deleting by delete gap settings.
- */
- public void setParams(String params) {
- super.setParams(params);
- docid = (int) Float.parseFloat(params);
- byStep = (docid < 0);
- }
+ }
+ }
+
+ /**
+ * Set the params (docid only)
+ * @param params docid to delete, or -1 for deleting by delete gap settings.
+ */
+ public void setParams(String params) {
+ super.setParams(params);
+ docid = (int) Float.parseFloat(params);
+ byStep = (docid < 0);
+ }
/* (non-Javadoc)
* @see org.apache.lucene.benchmark.byTask.tasks.PerfTask#supportsParams()
@@ -103,5 +103,5 @@
public boolean supportsParams() {
return true;
}
-
-}
+
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/DeleteDocTask.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewRoundTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewRoundTask.java?view=diff&rev=524969&r1=524968&r2=524969
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewRoundTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewRoundTask.java Mon Apr 2 16:37:14 2007
@@ -1,44 +1,44 @@
-package org.apache.lucene.benchmark.byTask.tasks;
-
-import org.apache.lucene.benchmark.byTask.PerfRunData;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-/**
- * Increment the counter for properties maintained by Round Number.
+package org.apache.lucene.benchmark.byTask.tasks;
+
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * Increment the counter for properties maintained by Round Number.
* <br>Other side effects: if there are props by round number, log value change.
- */
-public class NewRoundTask extends PerfTask {
-
- public NewRoundTask(PerfRunData runData) {
- super(runData);
- }
-
- public int doLogic() throws Exception {
- getRunData().getConfig().newRound();
- return 0;
- }
-
- /* (non-Javadoc)
- * @see PerfTask#shouldNotRecordStats()
- */
- protected boolean shouldNotRecordStats() {
- return true;
- }
-}
+ */
+public class NewRoundTask extends PerfTask {
+
+ public NewRoundTask(PerfRunData runData) {
+ super(runData);
+ }
+
+ public int doLogic() throws Exception {
+ getRunData().getConfig().newRound();
+ return 0;
+ }
+
+ /* (non-Javadoc)
+ * @see PerfTask#shouldNotRecordStats()
+ */
+ protected boolean shouldNotRecordStats() {
+ return true;
+ }
+}
Propchange: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/NewRoundTask.java
------------------------------------------------------------------------------
svn:eol-style = native