You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-user@lucene.apache.org by Jamie Johnson <je...@gmail.com> on 2015/08/07 01:35:30 UTC
Filtering documents using payloads
I am attempting to put together a DocsAndPositionsEnum that can hide terms
given the payload on the term. The idea is that if a term has a particular
access control and the user does not I don't want it to be visible. I have
based this off of
https://github.com/roshanp/lucure-core/blob/master/src/main/java/com/lucure/core/codec/AccessFilteredDocsAndPositionsEnum.java
with some modifications to try and preserve the position information that
is consumed as part of the hasAccess method. The current iteration that I
am working with seems to be providing wrong positions to the
ExactPhraseScorer.phraseFreq() method in the ChunkState's that are
calculated.
Below is my current iteration on this, but I can't narrow down why exactly
the position information isn't what I expect. Does anything jump out?
package com.lucure.core.codec;
import com.lucure.core.AuthorizationsHolder;
import com.lucure.core.security.Authorizations;
import com.lucure.core.security.FieldVisibility;
import com.lucure.core.security.VisibilityParseException;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
import java.util.Arrays;
import static com.lucure.core.codec.AccessFilteredDocsAndPositionsEnum
.AllAuthorizationsHolder.ALLAUTHSHOLDER;
/**
* Enum to read and restrict access to a document based on the payload which
* is expected to store the visibility
*/
public class AccessFilteredDocsAndPositionsEnum extends
DocsAndPositionsEnum {
/**
* This placeholder allows for lucene specific operations such as
* merge to read data with all authorizations enabled. This should never
* be used outside of the Codec.
*/
static class AllAuthorizationsHolder extends AuthorizationsHolder {
static final AllAuthorizationsHolder ALLAUTHSHOLDER = new
AllAuthorizationsHolder();
private AllAuthorizationsHolder() {
super(Authorizations.EMPTY);
}
}
static void enableMergeAuthorizations() {
AuthorizationsHolder.threadAuthorizations.set(ALLAUTHSHOLDER);
}
static void disableMergeAuthorizations() {
AuthorizationsHolder.threadAuthorizations.remove();
}
private final DocsAndPositionsEnum docsAndPositionsEnum;
private final AuthorizationsHolder authorizationsHolder;
public AccessFilteredDocsAndPositionsEnum(
DocsAndPositionsEnum docsAndPositionsEnum) {
this(docsAndPositionsEnum,
AuthorizationsHolder.threadAuthorizations.get());
}
public AccessFilteredDocsAndPositionsEnum(
DocsAndPositionsEnum docsAndPositionsEnum,
AuthorizationsHolder authorizationsHolder) {
this.docsAndPositionsEnum = docsAndPositionsEnum;
this.authorizationsHolder = authorizationsHolder;
}
long cost;
int endOffset, startOffset, currentPosition, freq, docId;
BytesRef payload;
@Override
public int nextPosition() throws IOException {
while (!hasAccess()) {
}
return currentPosition + 1;
}
@Override
public int startOffset() throws IOException {
return startOffset;
}
@Override
public int endOffset() throws IOException {
return endOffset;
}
@Override
public BytesRef getPayload() throws IOException {
return payload;
}
@Override
public int freq() throws IOException {
return docsAndPositionsEnum.freq();
}
@Override
public int docID() {
return docsAndPositionsEnum.docID();
}
@Override
public int nextDoc() throws IOException {
while (docsAndPositionsEnum.nextDoc() != NO_MORE_DOCS) {
if (hasAccess()) {
return docID();
}
}
return NO_MORE_DOCS;
}
@Override
public int advance(int target) throws IOException {
int advance = docsAndPositionsEnum.advance(target);
if (advance != NO_MORE_DOCS) {
if (hasAccess()) {
return docID();
} else {
//seek to next available
int doc;
while ((doc = nextDoc()) < target) {
}
return doc;
}
}
return NO_MORE_DOCS;
}
@Override
public long cost() {
return docsAndPositionsEnum.cost();
}
protected boolean hasAccess() throws IOException {
payload = docsAndPositionsEnum.getPayload();
endOffset = docsAndPositionsEnum.endOffset();
startOffset = docsAndPositionsEnum.startOffset();
currentPosition = docsAndPositionsEnum.nextPosition() - 1;
BytesRef payload = docsAndPositionsEnum.getPayload();
try {
if (payload == null ||
ALLAUTHSHOLDER.equals(authorizationsHolder) ||
this.authorizationsHolder.getVisibilityEvaluator().evaluate(
new
FieldVisibility(Arrays.copyOfRange(payload.bytes,
payload.offset,
payload.offset +
payload.length)))) {
return true;
}
} catch(VisibilityParseException e) {
}
return false;
}
@Override
public AttributeSource attributes() {
return super.attributes();
}
}
Re: Filtering documents using payloads
Posted by Jamie Johnson <je...@gmail.com>.
Looks like my issue is that my nextDoc call is consuming the first
position, and then on the call to nextPosition it's moving past where I
want it to be. I believe that I have this working properly now by checking
if the current position should or shouldn't be incremented.
On Thu, Aug 6, 2015 at 7:35 PM, Jamie Johnson <je...@gmail.com> wrote:
> I am attempting to put together a DocsAndPositionsEnum that can hide terms
> given the payload on the term. The idea is that if a term has a particular
> access control and the user does not I don't want it to be visible. I have
> based this off of
>
>
> https://github.com/roshanp/lucure-core/blob/master/src/main/java/com/lucure/core/codec/AccessFilteredDocsAndPositionsEnum.java
>
> with some modifications to try and preserve the position information that
> is consumed as part of the hasAccess method. The current iteration that I
> am working with seems to be providing wrong positions to the
> ExactPhraseScorer.phraseFreq() method in the ChunkState's that are
> calculated.
>
> Below is my current iteration on this, but I can't narrow down why exactly
> the position information isn't what I expect. Does anything jump out?
>
> package com.lucure.core.codec;
>
> import com.lucure.core.AuthorizationsHolder;
> import com.lucure.core.security.Authorizations;
> import com.lucure.core.security.FieldVisibility;
> import com.lucure.core.security.VisibilityParseException;
> import org.apache.lucene.index.DocsAndPositionsEnum;
> import org.apache.lucene.util.AttributeSource;
> import org.apache.lucene.util.BytesRef;
>
> import java.io.IOException;
> import java.util.Arrays;
>
> import static com.lucure.core.codec.AccessFilteredDocsAndPositionsEnum
> .AllAuthorizationsHolder.ALLAUTHSHOLDER;
>
> /**
> * Enum to read and restrict access to a document based on the payload
> which
> * is expected to store the visibility
> */
> public class AccessFilteredDocsAndPositionsEnum extends
> DocsAndPositionsEnum {
>
> /**
> * This placeholder allows for lucene specific operations such as
> * merge to read data with all authorizations enabled. This should
> never
> * be used outside of the Codec.
> */
> static class AllAuthorizationsHolder extends AuthorizationsHolder {
>
> static final AllAuthorizationsHolder ALLAUTHSHOLDER = new
> AllAuthorizationsHolder();
>
> private AllAuthorizationsHolder() {
> super(Authorizations.EMPTY);
> }
> }
>
> static void enableMergeAuthorizations() {
> AuthorizationsHolder.threadAuthorizations.set(ALLAUTHSHOLDER);
> }
>
> static void disableMergeAuthorizations() {
> AuthorizationsHolder.threadAuthorizations.remove();
> }
>
> private final DocsAndPositionsEnum docsAndPositionsEnum;
> private final AuthorizationsHolder authorizationsHolder;
>
> public AccessFilteredDocsAndPositionsEnum(
> DocsAndPositionsEnum docsAndPositionsEnum) {
> this(docsAndPositionsEnum,
> AuthorizationsHolder.threadAuthorizations.get());
> }
>
> public AccessFilteredDocsAndPositionsEnum(
> DocsAndPositionsEnum docsAndPositionsEnum,
> AuthorizationsHolder authorizationsHolder) {
> this.docsAndPositionsEnum = docsAndPositionsEnum;
> this.authorizationsHolder = authorizationsHolder;
> }
>
> long cost;
> int endOffset, startOffset, currentPosition, freq, docId;
> BytesRef payload;
>
> @Override
> public int nextPosition() throws IOException {
>
> while (!hasAccess()) {
>
> }
>
> return currentPosition + 1;
> }
>
> @Override
> public int startOffset() throws IOException {
> return startOffset;
> }
>
> @Override
> public int endOffset() throws IOException {
> return endOffset;
> }
>
> @Override
> public BytesRef getPayload() throws IOException {
> return payload;
> }
>
> @Override
> public int freq() throws IOException {
> return docsAndPositionsEnum.freq();
> }
>
> @Override
> public int docID() {
> return docsAndPositionsEnum.docID();
> }
>
> @Override
> public int nextDoc() throws IOException {
>
> while (docsAndPositionsEnum.nextDoc() != NO_MORE_DOCS) {
>
> if (hasAccess()) {
> return docID();
> }
> }
> return NO_MORE_DOCS;
>
> }
>
> @Override
> public int advance(int target) throws IOException {
> int advance = docsAndPositionsEnum.advance(target);
> if (advance != NO_MORE_DOCS) {
> if (hasAccess()) {
> return docID();
> } else {
> //seek to next available
> int doc;
> while ((doc = nextDoc()) < target) {
> }
> return doc;
> }
> }
> return NO_MORE_DOCS;
> }
>
> @Override
> public long cost() {
> return docsAndPositionsEnum.cost();
> }
>
> protected boolean hasAccess() throws IOException {
> payload = docsAndPositionsEnum.getPayload();
> endOffset = docsAndPositionsEnum.endOffset();
> startOffset = docsAndPositionsEnum.startOffset();
>
> currentPosition = docsAndPositionsEnum.nextPosition() - 1;
>
> BytesRef payload = docsAndPositionsEnum.getPayload();
> try {
> if (payload == null ||
> ALLAUTHSHOLDER.equals(authorizationsHolder) ||
>
> this.authorizationsHolder.getVisibilityEvaluator().evaluate(
> new
> FieldVisibility(Arrays.copyOfRange(payload.bytes,
> payload.offset,
> payload.offset +
> payload.length)))) {
> return true;
> }
> } catch(VisibilityParseException e) {
>
> }
> return false;
>
> }
>
> @Override
> public AttributeSource attributes() {
> return super.attributes();
> }
> }
>