You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by eh...@apache.org on 2005/12/28 14:57:48 UTC
svn commit: r359526 - in /lucene/java/trunk: contrib/regex/
contrib/regex/lib/ contrib/regex/src/ contrib/regex/src/java/
contrib/regex/src/java/org/ contrib/regex/src/java/org/apache/
contrib/regex/src/java/org/apache/lucene/ contrib/regex/src/java/or...
Author: ehatcher
Date: Wed Dec 28 05:57:40 2005
New Revision: 359526
URL: http://svn.apache.org/viewcvs?rev=359526&view=rev
Log:
Remove (Span)RegexQuery from core. Add completely refactored version to contrib/regex allowing pluggable regex implementations. contrib/regex is still a work in progress, and documentation is forthcoming
Added:
lucene/java/trunk/contrib/regex/
lucene/java/trunk/contrib/regex/build.xml
lucene/java/trunk/contrib/regex/lib/
lucene/java/trunk/contrib/regex/lib/jakarta-regexp-1.4.jar (with props)
lucene/java/trunk/contrib/regex/src/
lucene/java/trunk/contrib/regex/src/java/
lucene/java/trunk/contrib/regex/src/java/org/
lucene/java/trunk/contrib/regex/src/java/org/apache/
lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/
lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/
lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/
lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/JakartaRegexpCapabilities.java
lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/JavaUtilRegexCapabilities.java
lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexCapabilities.java
lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexQuery.java
lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexQueryCapable.java
lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexTermEnum.java
lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/SpanRegexQuery.java
lucene/java/trunk/contrib/regex/src/java/org/apache/regexp/
lucene/java/trunk/contrib/regex/src/java/org/apache/regexp/RegexpTunnel.java
lucene/java/trunk/contrib/regex/src/test/
lucene/java/trunk/contrib/regex/src/test/org/
lucene/java/trunk/contrib/regex/src/test/org/apache/
lucene/java/trunk/contrib/regex/src/test/org/apache/lucene/
lucene/java/trunk/contrib/regex/src/test/org/apache/lucene/search/
lucene/java/trunk/contrib/regex/src/test/org/apache/lucene/search/regex/
lucene/java/trunk/contrib/regex/src/test/org/apache/lucene/search/regex/TestRegexQuery.java
lucene/java/trunk/contrib/regex/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java
Removed:
lucene/java/trunk/src/java/org/apache/lucene/search/regex/
lucene/java/trunk/src/test/org/apache/lucene/search/regex/TestRegexQuery.java
lucene/java/trunk/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java
Added: lucene/java/trunk/contrib/regex/build.xml
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/regex/build.xml?rev=359526&view=auto
==============================================================================
--- lucene/java/trunk/contrib/regex/build.xml (added)
+++ lucene/java/trunk/contrib/regex/build.xml Wed Dec 28 05:57:40 2005
@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+
+<project name="regex" default="default">
+
+ <description>
+ Regular expression query
+ </description>
+
+ <path id="additional.dependencies">
+ <fileset dir="lib" includes="*-oro-*.jar,*-regexp-*.jar"/>
+ </path>
+
+ <pathconvert property="project.classpath"
+ targetos="unix"
+ refid="additional.dependencies"
+ />
+
+ <import file="../contrib-build.xml"/>
+</project>
Added: lucene/java/trunk/contrib/regex/lib/jakarta-regexp-1.4.jar
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/regex/lib/jakarta-regexp-1.4.jar?rev=359526&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/java/trunk/contrib/regex/lib/jakarta-regexp-1.4.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/JakartaRegexpCapabilities.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/JakartaRegexpCapabilities.java?rev=359526&view=auto
==============================================================================
--- lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/JakartaRegexpCapabilities.java (added)
+++ lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/JakartaRegexpCapabilities.java Wed Dec 28 05:57:40 2005
@@ -0,0 +1,21 @@
+package org.apache.lucene.search.regex;
+
+import org.apache.regexp.RE;
+import org.apache.regexp.RegexpTunnel;
+
+public class JakartaRegexpCapabilities implements RegexCapabilities {
+ private RE regexp;
+
+ public void compile(String pattern) {
+ regexp = new RE(pattern);
+ }
+
+ public boolean match(String string) {
+ return regexp.match(string);
+ }
+
+ public String prefix() {
+ char[] prefix = RegexpTunnel.getPrefix(regexp);
+ return prefix == null ? null : new String(prefix);
+ }
+}
Added: lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/JavaUtilRegexCapabilities.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/JavaUtilRegexCapabilities.java?rev=359526&view=auto
==============================================================================
--- lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/JavaUtilRegexCapabilities.java (added)
+++ lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/JavaUtilRegexCapabilities.java Wed Dec 28 05:57:40 2005
@@ -0,0 +1,19 @@
+package org.apache.lucene.search.regex;
+
+import java.util.regex.Pattern;
+
+public class JavaUtilRegexCapabilities implements RegexCapabilities {
+ private Pattern pattern;
+
+ public void compile(String pattern) {
+ this.pattern = Pattern.compile(pattern);
+ }
+
+ public boolean match(String string) {
+ return pattern.matcher(string).lookingAt();
+ }
+
+ public String prefix() {
+ return null;
+ }
+}
Added: lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexCapabilities.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexCapabilities.java?rev=359526&view=auto
==============================================================================
--- lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexCapabilities.java (added)
+++ lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexCapabilities.java Wed Dec 28 05:57:40 2005
@@ -0,0 +1,7 @@
+package org.apache.lucene.search.regex;
+
+public interface RegexCapabilities {
+ void compile(String pattern);
+ boolean match(String string);
+ String prefix();
+}
Added: lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexQuery.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexQuery.java?rev=359526&view=auto
==============================================================================
--- lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexQuery.java (added)
+++ lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexQuery.java Wed Dec 28 05:57:40 2005
@@ -0,0 +1,47 @@
+package org.apache.lucene.search.regex;
+
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.FilteredTermEnum;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.IndexReader;
+
+import java.io.IOException;
+
+public class RegexQuery extends MultiTermQuery implements RegexQueryCapable {
+ private RegexCapabilities regexImpl = new JavaUtilRegexCapabilities();
+
+ public RegexQuery(Term term) {
+ super(term);
+ }
+
+ public void setRegexImplementation(RegexCapabilities impl) {
+ this.regexImpl = impl;
+ }
+
+ public RegexCapabilities getRegexImplementation() {
+ return regexImpl;
+ }
+
+ protected FilteredTermEnum getEnum(IndexReader reader) throws IOException {
+ Term term = new Term(getTerm().field(), getTerm().text());
+ return new RegexTermEnum(reader, term, regexImpl);
+ }
+
+ /* generated by IntelliJ IDEA */
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ if (!super.equals(o)) return false;
+
+ final RegexQuery that = (RegexQuery) o;
+
+ return regexImpl.equals(that.regexImpl);
+ }
+
+ /* generated by IntelliJ IDEA */
+ public int hashCode() {
+ int result = super.hashCode();
+ result = 29 * result + regexImpl.hashCode();
+ return result;
+ }
+}
Added: lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexQueryCapable.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexQueryCapable.java?rev=359526&view=auto
==============================================================================
--- lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexQueryCapable.java (added)
+++ lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexQueryCapable.java Wed Dec 28 05:57:40 2005
@@ -0,0 +1,6 @@
+package org.apache.lucene.search.regex;
+
+public interface RegexQueryCapable {
+ void setRegexImplementation(RegexCapabilities impl);
+ RegexCapabilities getRegexImplementation();
+}
Added: lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexTermEnum.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexTermEnum.java?rev=359526&view=auto
==============================================================================
--- lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexTermEnum.java (added)
+++ lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/RegexTermEnum.java Wed Dec 28 05:57:40 2005
@@ -0,0 +1,53 @@
+package org.apache.lucene.search.regex;
+
+import org.apache.lucene.search.FilteredTermEnum;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+
+import java.io.IOException;
+
+public class RegexTermEnum extends FilteredTermEnum {
+ private String field = "";
+ private String pre = "";
+ private boolean endEnum = false;
+ private RegexCapabilities regexImpl;
+
+ public RegexTermEnum(IndexReader reader, Term term, RegexCapabilities regexImpl) throws IOException {
+ super();
+ field = term.field();
+ String text = term.text();
+ this.regexImpl = regexImpl;
+
+ regexImpl.compile(text);
+
+ pre = regexImpl.prefix();
+ if (pre == null) pre = "";
+
+ setEnum(reader.terms(new Term(term.field(), pre)));
+ }
+
+ protected final boolean termCompare(Term term) {
+ if (field == term.field()) {
+ String searchText = term.text();
+ if (searchText.startsWith(pre)) {
+ return regexImpl.match(searchText);
+ }
+ }
+ endEnum = true;
+ return false;
+ }
+
+ public final float difference() {
+// TODO: adjust difference based on distance of searchTerm.text() and term().text()
+ return 1.0f;
+ }
+
+ public final boolean endEnum() {
+ return endEnum;
+ }
+
+ public void close() throws IOException {
+ super.close();
+ field = null;
+ }
+}
Added: lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/SpanRegexQuery.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/SpanRegexQuery.java?rev=359526&view=auto
==============================================================================
--- lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/SpanRegexQuery.java (added)
+++ lucene/java/trunk/contrib/regex/src/java/org/apache/lucene/search/regex/SpanRegexQuery.java Wed Dec 28 05:57:40 2005
@@ -0,0 +1,105 @@
+package org.apache.lucene.search.regex;
+
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.spans.SpanOrQuery;
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.search.spans.Spans;
+import org.apache.lucene.util.ToStringUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.ArrayList;
+
+public class SpanRegexQuery extends SpanQuery implements RegexQueryCapable {
+ private RegexCapabilities regexImpl = new JavaUtilRegexCapabilities();
+ private Term term;
+
+ public SpanRegexQuery(Term term) {
+ this.term = term;
+ }
+
+ public Term getTerm() { return term; }
+
+ public Query rewrite(IndexReader reader) throws IOException {
+ RegexQuery orig = new RegexQuery(term);
+ orig.setRegexImplementation(regexImpl);
+
+ // RegexQuery (via MultiTermQuery).rewrite always returns a BooleanQuery
+ BooleanQuery bq = (BooleanQuery) orig.rewrite(reader);
+
+ BooleanClause[] clauses = bq.getClauses();
+ SpanQuery[] sqs = new SpanQuery[clauses.length];
+ for (int i = 0; i < clauses.length; i++) {
+ BooleanClause clause = clauses[i];
+
+ // Clauses from RegexQuery.rewrite are always TermQuery's
+ TermQuery tq = (TermQuery) clause.getQuery();
+
+ sqs[i] = new SpanTermQuery(tq.getTerm());
+ sqs[i].setBoost(tq.getBoost());
+ }
+
+ SpanOrQuery query = new SpanOrQuery(sqs);
+ query.setBoost(orig.getBoost());
+
+ return query;
+ }
+
+ public Spans getSpans(IndexReader reader) throws IOException {
+ throw new UnsupportedOperationException("Query should have been rewritten");
+ }
+
+ public String getField() {
+ return term.field();
+ }
+
+ public Collection getTerms() {
+ Collection terms = new ArrayList();
+ terms.add(term);
+ return terms;
+ }
+
+ /* generated by IntelliJ IDEA */
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ final SpanRegexQuery that = (SpanRegexQuery) o;
+
+ if (!regexImpl.equals(that.regexImpl)) return false;
+ if (!term.equals(that.term)) return false;
+
+ return true;
+ }
+
+ /* generated by IntelliJ IDEA */
+ public int hashCode() {
+ int result;
+ result = regexImpl.hashCode();
+ result = 29 * result + term.hashCode();
+ return result;
+ }
+
+ public String toString(String field) {
+ StringBuffer buffer = new StringBuffer();
+ buffer.append("spanRegexQuery(");
+ buffer.append(term);
+ buffer.append(")");
+ buffer.append(ToStringUtils.boost(getBoost()));
+ return buffer.toString();
+ }
+
+ public void setRegexImplementation(RegexCapabilities impl) {
+ this.regexImpl = impl;
+ }
+
+ public RegexCapabilities getRegexImplementation() {
+ return regexImpl;
+ }
+}
Added: lucene/java/trunk/contrib/regex/src/java/org/apache/regexp/RegexpTunnel.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/regex/src/java/org/apache/regexp/RegexpTunnel.java?rev=359526&view=auto
==============================================================================
--- lucene/java/trunk/contrib/regex/src/java/org/apache/regexp/RegexpTunnel.java (added)
+++ lucene/java/trunk/contrib/regex/src/java/org/apache/regexp/RegexpTunnel.java Wed Dec 28 05:57:40 2005
@@ -0,0 +1,11 @@
+package org.apache.regexp;
+
+/**
+ * This class exists as a gateway to access useful Jakarta Regexp package protected data.
+ */
+public class RegexpTunnel {
+ public static char[] getPrefix(RE regexp) {
+ REProgram program = regexp.getProgram();
+ return program.prefix;
+ }
+}
Added: lucene/java/trunk/contrib/regex/src/test/org/apache/lucene/search/regex/TestRegexQuery.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/regex/src/test/org/apache/lucene/search/regex/TestRegexQuery.java?rev=359526&view=auto
==============================================================================
--- lucene/java/trunk/contrib/regex/src/test/org/apache/lucene/search/regex/TestRegexQuery.java (added)
+++ lucene/java/trunk/contrib/regex/src/test/org/apache/lucene/search/regex/TestRegexQuery.java Wed Dec 28 05:57:40 2005
@@ -0,0 +1,101 @@
+package org.apache.lucene.search.regex;
+
+/**
+ * Copyright 2005 Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import junit.framework.TestCase;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.analysis.SimpleAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.search.IndexSearcher;
+
+import org.apache.lucene.search.spans.SpanNearQuery;
+import org.apache.lucene.search.spans.SpanQuery;
+
+public class TestRegexQuery extends TestCase {
+ private IndexSearcher searcher;
+ private final String FN = "field";
+
+ public void setUp() {
+ RAMDirectory directory = new RAMDirectory();
+ try {
+ IndexWriter writer = new IndexWriter(directory, new SimpleAnalyzer(), true);
+ Document doc = new Document();
+ doc.add(new Field(FN, "the quick brown fox jumps over the lazy dog", Field.Store.NO, Field.Index.TOKENIZED));
+ writer.addDocument(doc);
+ writer.optimize();
+ writer.close();
+ searcher = new IndexSearcher(directory);
+ } catch (Exception e) {
+ fail(e.toString());
+ }
+ }
+
+ public void tearDown() {
+ try {
+ searcher.close();
+ } catch (Exception e) {
+ fail(e.toString());
+ }
+ }
+
+ private Term newTerm(String value) { return new Term(FN, value); }
+
+ private int regexQueryNrHits(String regex) throws Exception {
+ RegexQuery query = new RegexQuery( newTerm(regex));
+ return searcher.search(query).length();
+ }
+
+ private int spanRegexQueryNrHits(String regex1, String regex2, int slop, boolean ordered) throws Exception {
+ SpanRegexQuery srq1 = new SpanRegexQuery( newTerm(regex1));
+ SpanRegexQuery srq2 = new SpanRegexQuery( newTerm(regex2));
+ SpanNearQuery query = new SpanNearQuery( new SpanQuery[]{srq1, srq2}, slop, ordered);
+ return searcher.search(query).length();
+ }
+
+ public void testRegex1() throws Exception {
+ assertEquals(1, regexQueryNrHits("^q.[aeiou]c.*$"));
+ }
+
+ public void testRegex2() throws Exception {
+ assertEquals(0, regexQueryNrHits("^.[aeiou]c.*$"));
+ }
+
+ public void testRegex3() throws Exception {
+ assertEquals(0, regexQueryNrHits("^q.[aeiou]c$"));
+ }
+
+ public void testSpanRegex1() throws Exception {
+ assertEquals(1, spanRegexQueryNrHits("^q.[aeiou]c.*$", "dog", 6, true));
+ }
+
+ public void testSpanRegex2() throws Exception {
+ assertEquals(0, spanRegexQueryNrHits("^q.[aeiou]c.*$", "dog", 5, true));
+ }
+
+ public void testEquals() throws Exception {
+ RegexQuery query1 = new RegexQuery( newTerm("foo.*"));
+ query1.setRegexImplementation(new JakartaRegexpCapabilities());
+
+ RegexQuery query2 = new RegexQuery( newTerm("foo.*"));
+ assertFalse(query1.equals(query2));
+ }
+
+}
+
Added: lucene/java/trunk/contrib/regex/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java
URL: http://svn.apache.org/viewcvs/lucene/java/trunk/contrib/regex/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java?rev=359526&view=auto
==============================================================================
--- lucene/java/trunk/contrib/regex/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java (added)
+++ lucene/java/trunk/contrib/regex/src/test/org/apache/lucene/search/regex/TestSpanRegexQuery.java Wed Dec 28 05:57:40 2005
@@ -0,0 +1,33 @@
+package org.apache.lucene.search.regex;
+
+import junit.framework.TestCase;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.analysis.SimpleAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.search.spans.SpanNearQuery;
+import org.apache.lucene.search.spans.SpanQuery;
+
+public class TestSpanRegexQuery extends TestCase {
+ public void testSpanRegex() throws Exception {
+ RAMDirectory directory = new RAMDirectory();
+ IndexWriter writer = new IndexWriter(directory, new SimpleAnalyzer(), true);
+ Document doc = new Document();
+ doc.add(new Field("field", "the quick brown fox jumps over the lazy dog", Field.Store.NO, Field.Index.TOKENIZED));
+ writer.addDocument(doc);
+ writer.optimize();
+ writer.close();
+
+ IndexSearcher searcher = new IndexSearcher(directory);
+ SpanRegexQuery srq = new SpanRegexQuery(new Term("field", "q.[aeiou]c.*"));
+ SpanTermQuery stq = new SpanTermQuery(new Term("field","dog"));
+ SpanNearQuery query = new SpanNearQuery(new SpanQuery[] {srq, stq}, 6, true);
+ Hits hits = searcher.search(query);
+ assertEquals(1, hits.length());
+ }
+}