[GitHub] [lucene-solr] romseygeek commented on a change in pull request #772: Payload-stored position length intervals

[GitBox <gi...@apache.org>]

romseygeek commented on a change in pull request #772: Payload-stored position length intervals
URL: https://github.com/apache/lucene-solr/pull/772#discussion_r302527832
 
 

 ##########
 File path: lucene/sandbox/src/java/org/apache/lucene/payloads/PayloadLengthTermIntervalsSource.java
 ##########
 @@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.payloads;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Objects;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.queries.intervals.IntervalIterator;
+import org.apache.lucene.queries.intervals.IntervalQuery;
+import org.apache.lucene.queries.intervals.IntervalsSource;
+import org.apache.lucene.search.MatchesIterator;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * An IntervalsSource that returns intervals over single terms, reading token
+ * length from each term's associated payload
+ *
+ * Tokens should be indexed using {@link PayloadTokenLengthFilter} to write
+ * token lengths into payloads
+ */
+public class PayloadLengthTermIntervalsSource extends IntervalsSource {
+
+  private final BytesRef term;
+
+  /**
+   * Create a PayloadLengthTermIntervalsSource for the given term
+   */
+  public PayloadLengthTermIntervalsSource(BytesRef term) {
+    this.term = term;
+  }
+
+  @Override
+  public IntervalIterator intervals(String field, LeafReaderContext ctx) throws IOException {
+    Terms terms = ctx.reader().terms(field);
+    if (terms == null)
+      return null;
+    if (terms.hasPositions() == false) {
+      throw new IllegalArgumentException("Cannot create an IntervalIterator over field " + field + " because it has no indexed positions");
+    }
+    TermsEnum te = terms.iterator();
+    if (te.seekExact(term) == false) {
+      return null;
+    }
+    return intervals(term, te);
+  }
+
+  private static int decodePayload(BytesRef payload) {
+    if (payload == null) {
+      return 1;
+    }
+    int length = 0;
+    for (int i = payload.offset; i < payload.offset + payload.length; i++) {
+      length = (length << 8) | Byte.toUnsignedInt(payload.bytes[i]);
+    }
+    return length;
+  }
+
+  private static IntervalIterator intervals(BytesRef term, TermsEnum te) throws IOException {
+    PostingsEnum pe = te.postings(null, PostingsEnum.PAYLOADS);
+    float cost = termPositionsCost(te);
+    return new IntervalIterator() {
+
+      int length;
+
+      @Override
+      public int docID() {
+        return pe.docID();
+      }
+
+      @Override
+      public int nextDoc() throws IOException {
+        int doc = pe.nextDoc();
+        reset();
+        return doc;
+      }
+
+      @Override
+      public int advance(int target) throws IOException {
+        int doc = pe.advance(target);
+        reset();
+        return doc;
+      }
+
+      @Override
+      public long cost() {
+        return pe.cost();
+      }
+
+      int pos = -1, upto;
+
+      @Override
+      public int start() {
+        return pos;
+      }
+
+      @Override
+      public int end() {
+        return pos + length - 1;
+      }
+
+      @Override
+      public int gaps() {
+        return 0;
+      }
+
+      @Override
+      public int nextInterval() throws IOException {
+        if (upto <= 0)
+          return pos = NO_MORE_INTERVALS;
+        upto--;
+        pos = pe.nextPosition();
+        length = decodePayload(pe.getPayload());
+        return pos;
+      }
+
+      @Override
+      public float matchCost() {
+        return cost;
+      }
+
+      private void reset() throws IOException {
+        if (pe.docID() == NO_MORE_DOCS) {
+          upto = -1;
+          pos = NO_MORE_INTERVALS;
+        }
+        else {
+          upto = pe.freq();
+          pos = -1;
+        }
+      }
+
+      @Override
+      public String toString() {
+        return term.utf8ToString() + ":" + super.toString();
+      }
+    };
+  }
+
+  @Override
+  public MatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
+    Terms terms = ctx.reader().terms(field);
+    if (terms == null)
+      return null;
+    if (terms.hasPositions() == false) {
+      throw new IllegalArgumentException("Cannot create an IntervalIterator over field " + field + " because it has no indexed positions");
+    }
+    TermsEnum te = terms.iterator();
+    if (te.seekExact(term) == false) {
+      return null;
+    }
+    return matches(te, doc);
+  }
+
+  private static MatchesIterator matches(TermsEnum te, int doc) throws IOException {
+    PostingsEnum pe = te.postings(null, PostingsEnum.ALL);
 
 Review comment:
   We need offsets for MatchesIterator

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org