You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2011/03/15 13:20:04 UTC
svn commit: r1081745 -
/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java
Author: dweiss
Date: Tue Mar 15 12:20:03 2011
New Revision: 1081745
URL: http://svn.apache.org/viewvc?rev=1081745&view=rev
Log:
Adding -noverify and a little bit nicer output to TestFSTs. These are debugging/analysis utils that are not used anywhere, so I commit them without the patch.
Modified:
lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java?rev=1081745&r1=1081744&r2=1081745&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java Tue Mar 15 12:20:03 2011
@@ -25,16 +25,7 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
+import java.util.*;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -1098,7 +1089,7 @@ public class TestFSTs extends LuceneTest
protected abstract T getOutput(IntsRef input, int ord) throws IOException;
- public void run(int limit) throws IOException {
+ public void run(int limit, boolean verify) throws IOException {
BufferedReader is = new BufferedReader(new InputStreamReader(new FileInputStream(wordsFileIn), "UTF-8"), 65536);
try {
final IntsRef intsRef = new IntsRef(10);
@@ -1115,7 +1106,9 @@ public class TestFSTs extends LuceneTest
ord++;
if (ord % 500000 == 0) {
- System.out.println(((System.currentTimeMillis()-tStart)/1000.0) + "s: " + ord + "...");
+ System.out.println(
+ String.format(Locale.ENGLISH,
+ "%6.2fs: %9d...", ((System.currentTimeMillis() - tStart) / 1000.0), ord));
}
if (ord >= limit) {
break;
@@ -1144,6 +1137,10 @@ public class TestFSTs extends LuceneTest
System.out.println("Saved FST to fst.bin.");
+ if (!verify) {
+ System.exit(0);
+ }
+
System.out.println("\nNow verify...");
is.close();
@@ -1194,6 +1191,7 @@ public class TestFSTs extends LuceneTest
int inputMode = 0; // utf8
boolean storeOrds = false;
boolean storeDocFreqs = false;
+ boolean verify = true;
while(idx < args.length) {
if (args[idx].equals("-prune")) {
prune = Integer.valueOf(args[1+idx]);
@@ -1215,6 +1213,9 @@ public class TestFSTs extends LuceneTest
if (args[idx].equals("-ords")) {
storeOrds = true;
}
+ if (args[idx].equals("-noverify")) {
+ verify = false;
+ }
idx++;
}
@@ -1235,7 +1236,7 @@ public class TestFSTs extends LuceneTest
return new PairOutputs.Pair<Long,Long>(o1.get(ord),
o2.get(_TestUtil.nextInt(rand, 1, 5000)));
}
- }.run(limit);
+ }.run(limit, verify);
} else if (storeOrds) {
// Store only ords
final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
@@ -1244,7 +1245,7 @@ public class TestFSTs extends LuceneTest
public Long getOutput(IntsRef input, int ord) {
return outputs.get(ord);
}
- }.run(limit);
+ }.run(limit, verify);
} else if (storeDocFreqs) {
// Store only docFreq
final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(false);
@@ -1257,7 +1258,7 @@ public class TestFSTs extends LuceneTest
}
return outputs.get(_TestUtil.nextInt(rand, 1, 5000));
}
- }.run(limit);
+ }.run(limit, verify);
} else {
// Store nothing
final NoOutputs outputs = NoOutputs.getSingleton();
@@ -1267,7 +1268,7 @@ public class TestFSTs extends LuceneTest
public Object getOutput(IntsRef input, int ord) {
return NO_OUTPUT;
}
- }.run(limit);
+ }.run(limit, verify);
}
}
Re: svn commit: r1081745 - /lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java
Posted by Dawid Weiss <da...@cs.put.poznan.pl>.
Thanks Mike :)
Dawid
On Tue, Mar 15, 2011 at 1:22 PM, Michael McCandless
<lu...@mikemccandless.com> wrote:
> Looks good Dawid!
>
> On Tue, Mar 15, 2011 at 8:20 AM, <dw...@apache.org> wrote:
>> Author: dweiss
>> Date: Tue Mar 15 12:20:03 2011
>> New Revision: 1081745
>>
>> URL: http://svn.apache.org/viewvc?rev=1081745&view=rev
>> Log:
>> Adding -noverify and a little bit nicer output to TestFSTs. These are debugging/analysis utils that are not used anywhere, so I commit them without the patch.
>>
>> Modified:
>> lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java
>>
>> Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java?rev=1081745&r1=1081744&r2=1081745&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java (original)
>> +++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java Tue Mar 15 12:20:03 2011
>> @@ -25,16 +25,7 @@ import java.io.IOException;
>> import java.io.InputStreamReader;
>> import java.io.OutputStreamWriter;
>> import java.io.Writer;
>> -import java.util.ArrayList;
>> -import java.util.Arrays;
>> -import java.util.Collections;
>> -import java.util.HashMap;
>> -import java.util.HashSet;
>> -import java.util.Iterator;
>> -import java.util.List;
>> -import java.util.Map;
>> -import java.util.Random;
>> -import java.util.Set;
>> +import java.util.*;
>>
>> import org.apache.lucene.analysis.MockAnalyzer;
>> import org.apache.lucene.document.Document;
>> @@ -1098,7 +1089,7 @@ public class TestFSTs extends LuceneTest
>>
>> protected abstract T getOutput(IntsRef input, int ord) throws IOException;
>>
>> - public void run(int limit) throws IOException {
>> + public void run(int limit, boolean verify) throws IOException {
>> BufferedReader is = new BufferedReader(new InputStreamReader(new FileInputStream(wordsFileIn), "UTF-8"), 65536);
>> try {
>> final IntsRef intsRef = new IntsRef(10);
>> @@ -1115,7 +1106,9 @@ public class TestFSTs extends LuceneTest
>>
>> ord++;
>> if (ord % 500000 == 0) {
>> - System.out.println(((System.currentTimeMillis()-tStart)/1000.0) + "s: " + ord + "...");
>> + System.out.println(
>> + String.format(Locale.ENGLISH,
>> + "%6.2fs: %9d...", ((System.currentTimeMillis() - tStart) / 1000.0), ord));
>> }
>> if (ord >= limit) {
>> break;
>> @@ -1144,6 +1137,10 @@ public class TestFSTs extends LuceneTest
>>
>> System.out.println("Saved FST to fst.bin.");
>>
>> + if (!verify) {
>> + System.exit(0);
>> + }
>> +
>> System.out.println("\nNow verify...");
>>
>> is.close();
>> @@ -1194,6 +1191,7 @@ public class TestFSTs extends LuceneTest
>> int inputMode = 0; // utf8
>> boolean storeOrds = false;
>> boolean storeDocFreqs = false;
>> + boolean verify = true;
>> while(idx < args.length) {
>> if (args[idx].equals("-prune")) {
>> prune = Integer.valueOf(args[1+idx]);
>> @@ -1215,6 +1213,9 @@ public class TestFSTs extends LuceneTest
>> if (args[idx].equals("-ords")) {
>> storeOrds = true;
>> }
>> + if (args[idx].equals("-noverify")) {
>> + verify = false;
>> + }
>> idx++;
>> }
>>
>> @@ -1235,7 +1236,7 @@ public class TestFSTs extends LuceneTest
>> return new PairOutputs.Pair<Long,Long>(o1.get(ord),
>> o2.get(_TestUtil.nextInt(rand, 1, 5000)));
>> }
>> - }.run(limit);
>> + }.run(limit, verify);
>> } else if (storeOrds) {
>> // Store only ords
>> final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
>> @@ -1244,7 +1245,7 @@ public class TestFSTs extends LuceneTest
>> public Long getOutput(IntsRef input, int ord) {
>> return outputs.get(ord);
>> }
>> - }.run(limit);
>> + }.run(limit, verify);
>> } else if (storeDocFreqs) {
>> // Store only docFreq
>> final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(false);
>> @@ -1257,7 +1258,7 @@ public class TestFSTs extends LuceneTest
>> }
>> return outputs.get(_TestUtil.nextInt(rand, 1, 5000));
>> }
>> - }.run(limit);
>> + }.run(limit, verify);
>> } else {
>> // Store nothing
>> final NoOutputs outputs = NoOutputs.getSingleton();
>> @@ -1267,7 +1268,7 @@ public class TestFSTs extends LuceneTest
>> public Object getOutput(IntsRef input, int ord) {
>> return NO_OUTPUT;
>> }
>> - }.run(limit);
>> + }.run(limit, verify);
>> }
>> }
>>
>>
>>
>>
>
>
>
> --
> Mike
>
> http://blog.mikemccandless.com
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: svn commit: r1081745 - /lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java
Posted by Michael McCandless <lu...@mikemccandless.com>.
Looks good Dawid!
On Tue, Mar 15, 2011 at 8:20 AM, <dw...@apache.org> wrote:
> Author: dweiss
> Date: Tue Mar 15 12:20:03 2011
> New Revision: 1081745
>
> URL: http://svn.apache.org/viewvc?rev=1081745&view=rev
> Log:
> Adding -noverify and a little bit nicer output to TestFSTs. These are debugging/analysis utils that are not used anywhere, so I commit them without the patch.
>
> Modified:
> lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java
>
> Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java?rev=1081745&r1=1081744&r2=1081745&view=diff
> ==============================================================================
> --- lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java (original)
> +++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java Tue Mar 15 12:20:03 2011
> @@ -25,16 +25,7 @@ import java.io.IOException;
> import java.io.InputStreamReader;
> import java.io.OutputStreamWriter;
> import java.io.Writer;
> -import java.util.ArrayList;
> -import java.util.Arrays;
> -import java.util.Collections;
> -import java.util.HashMap;
> -import java.util.HashSet;
> -import java.util.Iterator;
> -import java.util.List;
> -import java.util.Map;
> -import java.util.Random;
> -import java.util.Set;
> +import java.util.*;
>
> import org.apache.lucene.analysis.MockAnalyzer;
> import org.apache.lucene.document.Document;
> @@ -1098,7 +1089,7 @@ public class TestFSTs extends LuceneTest
>
> protected abstract T getOutput(IntsRef input, int ord) throws IOException;
>
> - public void run(int limit) throws IOException {
> + public void run(int limit, boolean verify) throws IOException {
> BufferedReader is = new BufferedReader(new InputStreamReader(new FileInputStream(wordsFileIn), "UTF-8"), 65536);
> try {
> final IntsRef intsRef = new IntsRef(10);
> @@ -1115,7 +1106,9 @@ public class TestFSTs extends LuceneTest
>
> ord++;
> if (ord % 500000 == 0) {
> - System.out.println(((System.currentTimeMillis()-tStart)/1000.0) + "s: " + ord + "...");
> + System.out.println(
> + String.format(Locale.ENGLISH,
> + "%6.2fs: %9d...", ((System.currentTimeMillis() - tStart) / 1000.0), ord));
> }
> if (ord >= limit) {
> break;
> @@ -1144,6 +1137,10 @@ public class TestFSTs extends LuceneTest
>
> System.out.println("Saved FST to fst.bin.");
>
> + if (!verify) {
> + System.exit(0);
> + }
> +
> System.out.println("\nNow verify...");
>
> is.close();
> @@ -1194,6 +1191,7 @@ public class TestFSTs extends LuceneTest
> int inputMode = 0; // utf8
> boolean storeOrds = false;
> boolean storeDocFreqs = false;
> + boolean verify = true;
> while(idx < args.length) {
> if (args[idx].equals("-prune")) {
> prune = Integer.valueOf(args[1+idx]);
> @@ -1215,6 +1213,9 @@ public class TestFSTs extends LuceneTest
> if (args[idx].equals("-ords")) {
> storeOrds = true;
> }
> + if (args[idx].equals("-noverify")) {
> + verify = false;
> + }
> idx++;
> }
>
> @@ -1235,7 +1236,7 @@ public class TestFSTs extends LuceneTest
> return new PairOutputs.Pair<Long,Long>(o1.get(ord),
> o2.get(_TestUtil.nextInt(rand, 1, 5000)));
> }
> - }.run(limit);
> + }.run(limit, verify);
> } else if (storeOrds) {
> // Store only ords
> final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
> @@ -1244,7 +1245,7 @@ public class TestFSTs extends LuceneTest
> public Long getOutput(IntsRef input, int ord) {
> return outputs.get(ord);
> }
> - }.run(limit);
> + }.run(limit, verify);
> } else if (storeDocFreqs) {
> // Store only docFreq
> final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(false);
> @@ -1257,7 +1258,7 @@ public class TestFSTs extends LuceneTest
> }
> return outputs.get(_TestUtil.nextInt(rand, 1, 5000));
> }
> - }.run(limit);
> + }.run(limit, verify);
> } else {
> // Store nothing
> final NoOutputs outputs = NoOutputs.getSingleton();
> @@ -1267,7 +1268,7 @@ public class TestFSTs extends LuceneTest
> public Object getOutput(IntsRef input, int ord) {
> return NO_OUTPUT;
> }
> - }.run(limit);
> + }.run(limit, verify);
> }
> }
>
>
>
>
--
Mike
http://blog.mikemccandless.com
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org