You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2011/12/22 20:59:31 UTC
svn commit: r1222413 - in /incubator/accumulo/trunk/contrib/accumulo_sample:
./ query/src/main/java/iterator/ query/src/main/java/logic/
query/src/main/java/sample/query/
Author: ecn
Date: Thu Dec 22 19:59:30 2011
New Revision: 1222413
URL: http://svn.apache.org/viewvc?rev=1222413&view=rev
Log:
ACCUMULO-230
Fixed key decoding in iterators.
Fixed restart of query in AndIterator
Fixed the detection of scan/seek in OptimizedQueryIterator
Added code to skip over the Field Index within the tablet
Modified:
incubator/accumulo/trunk/contrib/accumulo_sample/pom.xml
incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AbstractEvaluatingIterator.java
incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AndIterator.java
incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/BooleanLogicIterator.java
incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/EvaluatingIterator.java
incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OptimizedQueryIterator.java
incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OrIterator.java
incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/logic/AbstractQueryLogic.java
incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/sample/query/Query.java
Modified: incubator/accumulo/trunk/contrib/accumulo_sample/pom.xml
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/pom.xml?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/pom.xml (original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/pom.xml Thu Dec 22 19:59:30 2011
@@ -187,7 +187,7 @@
<version.lucene-wikipedia>3.0.2</version.lucene-wikipedia>
<version.protobuf>2.3.0</version.protobuf>
<version.googlecollections>1.0</version.googlecollections>
- <version.libthrift>0.8.0</version.libthrift>
+ <version.libthrift>0.6.1</version.libthrift>
<version.zookeeper>3.3.1</version.zookeeper>
<version.minlog>1.2</version.minlog>
</properties>
Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AbstractEvaluatingIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AbstractEvaluatingIterator.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AbstractEvaluatingIterator.java (original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AbstractEvaluatingIterator.java Thu Dec 22 19:59:30 2011
@@ -25,11 +25,6 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.jexl2.parser.ParseException;
-import org.apache.log4j.Logger;
-
-import parser.EventFields;
-import parser.QueryEvaluator;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
@@ -38,6 +33,11 @@ import org.apache.accumulo.core.data.Val
import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.OptionDescriber;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.commons.jexl2.parser.ParseException;
+import org.apache.log4j.Logger;
+
+import parser.EventFields;
+import parser.QueryEvaluator;
import com.esotericsoftware.kryo.Kryo;
@@ -66,7 +66,7 @@ public abstract class AbstractEvaluating
public static final String UNEVALUTED_EXPRESSIONS = "unevaluated.expressions";
private PartialKey comparator = null;
- private SortedKeyValueIterator<Key,Value> iterator;
+ protected SortedKeyValueIterator<Key,Value> iterator;
private Key currentKey = new Key();
private Key returnKey;
private Value returnValue;
@@ -118,8 +118,9 @@ public abstract class AbstractEvaluating
*
* @param key
* @return
+ * @throws IOException
*/
- public abstract boolean isKeyAccepted(Key key);
+ public abstract boolean isKeyAccepted(Key key) throws IOException;
/**
* Reset state.
Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AndIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AndIterator.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AndIterator.java (original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AndIterator.java Thu Dec 22 19:59:30 2011
@@ -140,7 +140,7 @@ public class AndIterator implements Sort
int idx = 0;
String sKey = key.getColumnQualifier().toString();
- idx = sKey.lastIndexOf("\0");
+ idx = sKey.indexOf("\0");
return sKey.substring(idx + 1);
}
@@ -781,15 +781,16 @@ public class AndIterator implements Sort
Key sourceKey;
if (range.getStartKey() != null) {
// Build a key with the DocID if one is given
- if (range.getStartKey().getColumnQualifier() != null) {
+ if (range.getStartKey().getColumnFamily() != null) {
sourceKey = buildKey(getPartition(range.getStartKey()), (sources[i].dataLocation == null) ? nullText : sources[i].dataLocation,
- (sources[i].term == null) ? nullText : new Text(sources[i].term + "\0" + range.getStartKey().getColumnQualifier()));
+ (sources[i].term == null) ? nullText : new Text(sources[i].term + "\0" + range.getStartKey().getColumnFamily()));
} // Build a key with just the term.
else {
sourceKey = buildKey(getPartition(range.getStartKey()), (sources[i].dataLocation == null) ? nullText : sources[i].dataLocation,
(sources[i].term == null) ? nullText : sources[i].term);
}
-
+ if (!range.isStartKeyInclusive())
+ sourceKey = sourceKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL);
sources[i].iter.seek(new Range(sourceKey, true, null, false), seekColumnFamilies, inclusive);
} else {
sources[i].iter.seek(range, seekColumnFamilies, inclusive);
@@ -889,7 +890,7 @@ public class AndIterator implements Sort
return hasTop();
} else {
// need to check uid
- String myUid = getUID(this.topKey);
+ String myUid = this.topKey.getColumnQualifier().toString();
String jumpUid = getUID(jumpKey);
if (log.isDebugEnabled()) {
if (myUid == null) {
Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/BooleanLogicIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/BooleanLogicIterator.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/BooleanLogicIterator.java (original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/BooleanLogicIterator.java Thu Dec 22 19:59:30 2011
@@ -1204,16 +1204,10 @@ public class BooleanLogicIterator implem
}
private String getEventKeyUid(Key k) {
- String uid = null;
if (k == null || k.getColumnFamily() == null) {
- return uid;
+ return null;
} else {
- String[] parts = k.getColumnFamily().toString().split("\0");
- if (parts.length < 2) {
- return uid;
- } else {
- return parts[parts.length - 1];
- }
+ return k.getColumnFamily().toString();
}
}
@@ -1221,7 +1215,7 @@ public class BooleanLogicIterator implem
try {
int idx = 0;
String sKey = k.getColumnQualifier().toString();
- idx = sKey.lastIndexOf("\0");
+ idx = sKey.indexOf("\0");
return sKey.substring(idx + 1);
} catch (Exception e) {
return null;
@@ -1865,6 +1859,8 @@ public class BooleanLogicIterator implem
}
}
+ log.debug("overallRange " + overallRange + " topKey " + this.root.getTopKey() + " contains " + overallRange.contains(this.root.getTopKey()));
+
if (overallRange.contains(this.root.getTopKey())) {
setTopKey(this.root.getTopKey());
} else {
Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/EvaluatingIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/EvaluatingIterator.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/EvaluatingIterator.java (original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/EvaluatingIterator.java Thu Dec 22 19:59:30 2011
@@ -16,14 +16,22 @@
*/
package iterator;
-import parser.EventFields;
-import parser.EventFields.FieldValue;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.hadoop.io.Text;
+
+import parser.EventFields;
+import parser.EventFields.FieldValue;
public class EvaluatingIterator extends AbstractEvaluatingIterator {
@@ -77,9 +85,14 @@ public class EvaluatingIterator extends
* Don't accept this key if the colf starts with 'fi'
*/
@Override
- public boolean isKeyAccepted(Key key) {
+ public boolean isKeyAccepted(Key key) throws IOException {
if (key.getColumnFamily().toString().startsWith("fi")) {
- return false;
+ Key copy = new Key(key.getRow(), new Text("fi\01"));
+ Collection<ByteSequence> columnFamilies = Collections.emptyList();
+ this.iterator.seek(new Range(copy, copy), columnFamilies, true);
+ if (this.iterator.hasTop())
+ return isKeyAccepted(this.iterator.getTopKey());
+ return true;
}
return true;
}
Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OptimizedQueryIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OptimizedQueryIterator.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OptimizedQueryIterator.java (original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OptimizedQueryIterator.java Thu Dec 22 19:59:30 2011
@@ -22,15 +22,15 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
-import org.apache.log4j.Logger;
-
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.OptionDescriber;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.log4j.Logger;
/**
* This iterator internally uses the BooleanLogicIterator to find event UIDs in the field index portion of the partition and uses the EvaluatingIterator to
@@ -44,6 +44,7 @@ public class OptimizedQueryIterator impl
private Key key = null;
private Value value = null;
private boolean eventSpecificRange = false;
+ private long seekTime = 0;
public IteratorOptions describeOptions() {
Map<String,String> options = new HashMap<String,String>();
@@ -139,7 +140,9 @@ public class OptimizedQueryIterator impl
// If the index has a match, then seek the event to the key
if (index.hasTop()) {
Key eventKey = index.getTopKey();
- Range eventRange = new Range(eventKey.getRow());
+ Key endKey = eventKey.followingKey(PartialKey.ROW_COLFAM);
+ Key startKey = new Key(eventKey.getRow(), eventKey.getColumnFamily());
+ Range eventRange = new Range(startKey, endKey);
HashSet<ByteSequence> cf = new HashSet<ByteSequence>();
cf.add(eventKey.getColumnFamilyData());
event.seek(eventRange, cf, true);
@@ -164,7 +167,7 @@ public class OptimizedQueryIterator impl
log.debug("seek, range:" + range);
}
// Test the range to see if it is event specific.
- if (null != range.getStartKey() && range.getStartKey().getColumnFamily() != null && !range.getStartKey().getColumnFamily().toString().equals("")) {
+ if (null != range.getEndKey() && range.getEndKey().getColumnFamily() != null && range.getEndKey().getColumnFamily().getLength() != 0) {
if (log.isDebugEnabled()) {
log.debug("Jumping straight to the event");
}
Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OrIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OrIterator.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OrIterator.java (original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OrIterator.java Thu Dec 22 19:59:30 2011
@@ -23,9 +23,6 @@ import java.util.Iterator;
import java.util.Map;
import java.util.PriorityQueue;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
-
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
@@ -33,6 +30,8 @@ import org.apache.accumulo.core.data.Ran
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
/**
* An iterator that handles "OR" query constructs on the server side. This code has been adapted/merged from Heap and Multi Iterators.
@@ -206,7 +205,7 @@ public class OrIterator implements Sorte
int idx = 0;
String sKey = key.getColumnQualifier().toString();
- idx = sKey.lastIndexOf("\0");
+ idx = sKey.indexOf("\0");
return sKey.substring(idx + 1);
} catch (Exception e) {
return null;
Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/logic/AbstractQueryLogic.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/logic/AbstractQueryLogic.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/logic/AbstractQueryLogic.java (original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/logic/AbstractQueryLogic.java Thu Dec 22 19:59:30 2011
@@ -58,7 +58,6 @@ import parser.EventFields;
import parser.EventFields.FieldValue;
import parser.FieldIndexQueryReWriter;
import parser.JexlOperatorConstants;
-import parser.JexlOperatorConstants;
import parser.QueryParser;
import parser.QueryParser.QueryTerm;
import parser.RangeCalculator;
@@ -460,7 +459,8 @@ public abstract class AbstractQueryLogic
Set<Range> ranges = new HashSet<Range>();
Set<String> typeFilter = types;
- Authorizations auths = new Authorizations(StringUtils.join(authorizations, "|"));
+ String array[] = authorizations.toArray(new String[0]);
+ Authorizations auths = new Authorizations(array);
Results results = new Results();
// Get the query string
@@ -785,9 +785,6 @@ public abstract class AbstractQueryLogic
long count = 0;
for (Entry<Key,Value> entry : bs) {
count++;
- if (log.isDebugEnabled()) {
- log.debug("Key: " + entry.getKey());// + "\nValue: " + entry.getValue() + "\n");
- }
// The key that is returned by the EvaluatingIterator is not the same key that is in
// the table. The value that is returned by the EvaluatingIterator is a kryo
// serialized EventFields object.
Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/sample/query/Query.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/sample/query/Query.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/sample/query/Query.java (original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/sample/query/Query.java Thu Dec 22 19:59:30 2011
@@ -192,9 +192,12 @@ public class Query implements IQuery {
// Create list of auths
List<String> authorizations = new ArrayList<String>();
- for (String a : auths.split(","))
- authorizations.add(a);
-
+ if (auths == null || "".equals(auths)) {
+ authorizations.add("all");
+ } else {
+ for (String a : auths.split(","))
+ authorizations.add(a);
+ }
ContentLogic table = new ContentLogic();
table.setTableName(tableName);
return table.runQuery(connector, query, authorizations);