You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/01/29 17:20:35 UTC
[1/9] git commit: Fixes export class path issue
Updated Branches:
refs/heads/USERGRID-2862-limitfix [deleted] acc95d131
refs/heads/master 391e7f4dc -> c36f5ae68
refs/pull/41/merge [deleted] 31dbeed90
Fixes export class path issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b5492f97
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b5492f97
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b5492f97
Branch: refs/heads/master
Commit: b5492f979332a0fc4866cea1c9a663dcccfb0b66
Parents: 832017b
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Jan 21 13:45:15 2014 -0800
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jan 21 13:45:15 2014 -0800
----------------------------------------------------------------------
stack/tools/pom.xml | 4 ++++
.../src/main/java/org/usergrid/tools/bean/ExportOrg.java | 11 +++++++++++
2 files changed, 15 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5492f97/stack/tools/pom.xml
----------------------------------------------------------------------
diff --git a/stack/tools/pom.xml b/stack/tools/pom.xml
index fb5fe95..385303b 100644
--- a/stack/tools/pom.xml
+++ b/stack/tools/pom.xml
@@ -193,6 +193,10 @@
<version>9.1-901.jdbc4</version>
</dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </dependency>
<!-- Testing and Logging Dependencies -->
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5492f97/stack/tools/src/main/java/org/usergrid/tools/bean/ExportOrg.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/usergrid/tools/bean/ExportOrg.java b/stack/tools/src/main/java/org/usergrid/tools/bean/ExportOrg.java
index bb78c1e..fc5a21b 100644
--- a/stack/tools/src/main/java/org/usergrid/tools/bean/ExportOrg.java
+++ b/stack/tools/src/main/java/org/usergrid/tools/bean/ExportOrg.java
@@ -26,6 +26,7 @@ import org.usergrid.management.OrganizationInfo;
public class ExportOrg extends OrganizationInfo {
private List<String> adminUserNames;
+ private int passwordHistorySize;
public ExportOrg() {
@@ -55,4 +56,14 @@ public class ExportOrg extends OrganizationInfo {
public void addAdmin( String username ) {
adminUserNames.add( username );
}
+
+
+ public int getPasswordHistorySize() {
+ return passwordHistorySize;
+ }
+
+
+ public void setPasswordHistorySize( final int passwordHistorySize ) {
+ this.passwordHistorySize = passwordHistorySize;
+ }
}
[7/9] git commit: Fixes query processor size and test to work
correctly with the 1k defaults
Posted by sn...@apache.org.
Fixes query processor size and test to work correctly with the 1k defaults
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b05edcdf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b05edcdf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b05edcdf
Branch: refs/heads/master
Commit: b05edcdfb26606965c886975d05e344ee590ab51
Parents: 8d4425d
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Jan 28 18:41:21 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jan 28 18:41:21 2014 -0700
----------------------------------------------------------------------
.../org/usergrid/persistence/cassandra/QueryProcessor.java | 3 +--
.../usergrid/persistence/query/IntersectionUnionPagingIT.java | 6 ++++--
stack/core/src/test/resources/log4j.properties | 2 +-
3 files changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b05edcdf/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java b/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java
index 4389b79..5d9bbc8 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java
@@ -77,8 +77,7 @@ import static org.usergrid.persistence.Schema.getDefaultSchema;
public class QueryProcessor {
- private static final int PAGE_SIZE = 10;
-// private static final int PAGE_SIZE = 1000;
+ public static final int PAGE_SIZE = 1000;
private static final Logger logger = LoggerFactory.getLogger( QueryProcessor.class );
private static final Schema SCHEMA = getDefaultSchema();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b05edcdf/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java b/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java
index 76995b9..94fdc40 100644
--- a/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java
+++ b/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java
@@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory;
import org.usergrid.persistence.Entity;
import org.usergrid.persistence.Query;
import org.usergrid.persistence.Results;
+import org.usergrid.persistence.cassandra.QueryProcessor;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -44,7 +45,8 @@ public class IntersectionUnionPagingIT extends AbstractIteratingQueryIT {
"select * where (field1Or > '00000000' OR field2Or > '00000000') AND fieldDate = '0000-00-00'";
private static final String scanUnion =
"select * where fieldDate = '0000-00-00' AND (field1Or > '00000000' OR field2Or > '00000000') ";
- private static final int PAGE_SIZE = 20;
+
+ private static final int PAGE_SIZE = 300;
@Test
@@ -77,7 +79,7 @@ public class IntersectionUnionPagingIT extends AbstractIteratingQueryIT {
private Set<String> performSetup( final IoHelper io ) throws Exception {
io.doSetup();
- int size = 40;
+ int size = ( int ) ( QueryProcessor.PAGE_SIZE*2.5);
long start = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b05edcdf/stack/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/log4j.properties b/stack/core/src/test/resources/log4j.properties
index 9282771..4c75404 100644
--- a/stack/core/src/test/resources/log4j.properties
+++ b/stack/core/src/test/resources/log4j.properties
@@ -18,7 +18,7 @@
# and the pattern to %c instead of %l. (%l is slower.)
# output messages into a rolling log file as well as stdout
-log4j.rootLogger=DEBUG,stdout
+log4j.rootLogger=INFO,stdout
# stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
[5/9] git commit: Fixes advance logic and page count in merge
iterator. Fixes reset in Union iterator
Posted by sn...@apache.org.
Fixes advance logic and page count in merge iterator. Fixes reset in Union iterator
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2303624a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2303624a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2303624a
Branch: refs/heads/master
Commit: 2303624a1e428201072f5fce1464dd2a45b07d36
Parents: 16455b8
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Jan 24 13:01:33 2014 -0800
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jan 28 18:36:11 2014 -0700
----------------------------------------------------------------------
.../query/ir/result/MergeIterator.java | 19 +++--
.../query/ir/result/UnionIterator.java | 15 ++++
.../query/ir/result/UnionIteratorTest.java | 88 ++++++++++++++++++++
3 files changed, 114 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2303624a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java
index 46807e1..e09cff5 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java
@@ -62,24 +62,27 @@ public abstract class MergeIterator implements ResultIterator {
@Override
public boolean hasNext() {
//if next isn't set, try to advance
- if ( next == null ) {
- doAdvance();
+ if(next != null){
+ return true;
}
- boolean results = next != null && next.size() > 0;
- if ( results ) {
- last = next;
- loadCount++;
- }
+ doAdvance();
+
- return results;
+ return next != null;
}
/** Advance to the next page */
protected void doAdvance() {
next = advance();
+
+
+ if ( next != null && next.size() > 0 ) {
+ last = next;
+ loadCount++;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2303624a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java
index c7b31d1..01daf36 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java
@@ -120,6 +120,16 @@ public class UnionIterator extends MultiIterator {
}
+ @Override
+ public void doReset() {
+ //reset sub iterators if we need to
+ super.doReset();
+
+ list.reset();
+
+ }
+
+
/**
* A Sorted Set with a max size. When a new entry is added, the max is removed. You can mark the next "min" by
* calling the mark method. Values > min are accepted. Values > min and that are over size are discarded
@@ -224,6 +234,11 @@ public class UnionIterator extends MultiIterator {
public void clear() {
this.list.clear();
}
+
+ public void reset(){
+ clear();
+ this.min = null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2303624a/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/UnionIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/UnionIteratorTest.java b/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/UnionIteratorTest.java
index 8fd9ea8..24851e3 100644
--- a/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/UnionIteratorTest.java
+++ b/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/UnionIteratorTest.java
@@ -365,6 +365,94 @@ public class UnionIteratorTest {
}
+ @Test
+ public void resetCorrect() {
+
+ UUID id1 = UUIDUtils.minTimeUUID( 1 );
+ UUID id2 = UUIDUtils.minTimeUUID( 2 );
+ UUID id3 = UUIDUtils.minTimeUUID( 3 );
+ UUID id4 = UUIDUtils.minTimeUUID( 4 );
+ UUID id5 = UUIDUtils.minTimeUUID( 5 );
+ UUID id6 = UUIDUtils.minTimeUUID( 6 );
+ UUID id7 = UUIDUtils.minTimeUUID( 75 );
+
+
+ UnionIterator union = new UnionIterator( 5, 0, null );
+
+ InOrderIterator first = new InOrderIterator( 100 );
+ first.add( id3 );
+ first.add( id6 );
+ first.add( id4 );
+
+
+ InOrderIterator second = new InOrderIterator( 100 );
+ second.add( id7 );
+ second.add( id1 );
+ second.add( id2 );
+ second.add( id5 );
+
+
+ union.addIterator( first );
+ union.addIterator( second );
+
+
+ // now make sure it's right, only 1, 3 and 8 intersect
+ assertTrue( union.hasNext() );
+
+ Set<ScanColumn> ids = union.next();
+
+
+ assertEquals(5, ids.size());
+
+ // now make sure it's right, only 1, 3 and 8 intersect
+ assertTrue( ids.contains( uuidColumn( id1 ) ) );
+ assertTrue( ids.contains( uuidColumn( id2 ) ) );
+ assertTrue( ids.contains( uuidColumn( id3 ) ) );
+ assertTrue( ids.contains( uuidColumn( id4 ) ) );
+ assertTrue( ids.contains( uuidColumn( id5 ) ) );
+
+ ids = union.next();
+
+
+ assertEquals(2, ids.size());
+
+ assertTrue( ids.contains( uuidColumn( id6 ) ) );
+ assertTrue( ids.contains( uuidColumn( id7 ) ) );
+
+ //now try to get the next page
+ ids = union.next();
+ assertNull( ids );
+
+ //now reset and re-test
+ union.reset();
+
+ ids = union.next();
+
+ assertEquals(5, ids.size());
+
+
+ // now make sure it's right, only 1, 3 and 8 intersect
+ assertTrue( ids.contains( uuidColumn( id1 ) ) );
+ assertTrue( ids.contains( uuidColumn( id2 ) ) );
+ assertTrue( ids.contains( uuidColumn( id3 ) ) );
+ assertTrue( ids.contains( uuidColumn( id4 ) ) );
+ assertTrue( ids.contains( uuidColumn( id5 ) ) );
+
+
+ ids = union.next();
+
+ assertEquals(2, ids.size());
+
+ assertTrue( ids.contains( uuidColumn( id6 ) ) );
+ assertTrue( ids.contains( uuidColumn( id7 ) ) );
+
+
+ //now try to get the next page
+ ids = union.next();
+ assertNull( ids );
+ }
+
+
private void reverse( UUID[] array ) {
UUID temp = null;
[3/9] git commit: Changed page size to create issue. Need to update
test to work with page size of 1k to create the problem
Posted by sn...@apache.org.
Changed page size to create issue. Need to update test to work with page size of 1k to create the problem
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/16455b87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/16455b87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/16455b87
Branch: refs/heads/master
Commit: 16455b87386f7d2aefb7e70cef5140ea3cf160fb
Parents: ba1f456
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Jan 22 10:48:42 2014 -0800
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jan 28 18:35:37 2014 -0700
----------------------------------------------------------------------
.../persistence/cassandra/QueryProcessor.java | 3 ++-
.../query/ir/result/IntersectionIterator.java | 4 +++-
.../query/ir/result/SliceIterator.java | 3 +++
.../query/IntersectionUnionPagingIT.java | 18 ++++++++++--------
.../query/ir/result/IntersectionIteratorTest.java | 6 ++++--
stack/core/src/test/resources/log4j.properties | 2 +-
6 files changed, 23 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/16455b87/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java b/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java
index b5d77a3..4389b79 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/cassandra/QueryProcessor.java
@@ -77,7 +77,8 @@ import static org.usergrid.persistence.Schema.getDefaultSchema;
public class QueryProcessor {
- private static final int PAGE_SIZE = 1000;
+ private static final int PAGE_SIZE = 10;
+// private static final int PAGE_SIZE = 1000;
private static final Logger logger = LoggerFactory.getLogger( QueryProcessor.class );
private static final Schema SCHEMA = getDefaultSchema();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/16455b87/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/IntersectionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/IntersectionIterator.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/IntersectionIterator.java
index 4ffad1b..5948cb9 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/IntersectionIterator.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/IntersectionIterator.java
@@ -141,7 +141,9 @@ public class IntersectionIterator extends MultiIterator {
final Set<ScanColumn> childResults = child.next();
- results.addAll( Sets.intersection( current, childResults ) );
+ final Set<ScanColumn> intersection = Sets.intersection( current, childResults );
+
+ results.addAll( intersection );
}
return results;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/16455b87/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
index 855a844..6922b9f 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
@@ -142,6 +142,9 @@ public class SliceIterator implements ResultIterator {
continue;
}
+
+ logger.debug( "Parsed column with uuid '{}'", parsed.getUUID() );
+
last = parsed;
cols.put( parsed.getUUID(), parsed );
parsedCols.add( parsed );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/16455b87/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java b/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java
index 5f554d3..76995b9 100644
--- a/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java
+++ b/stack/core/src/test/java/org/usergrid/persistence/query/IntersectionUnionPagingIT.java
@@ -25,6 +25,7 @@ import java.util.Set;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.usergrid.persistence.Entity;
import org.usergrid.persistence.Query;
import org.usergrid.persistence.Results;
@@ -43,7 +44,7 @@ public class IntersectionUnionPagingIT extends AbstractIteratingQueryIT {
"select * where (field1Or > '00000000' OR field2Or > '00000000') AND fieldDate = '0000-00-00'";
private static final String scanUnion =
"select * where fieldDate = '0000-00-00' AND (field1Or > '00000000' OR field2Or > '00000000') ";
- private static final int PAGE_SIZE = 100;
+ private static final int PAGE_SIZE = 20;
@Test
@@ -55,7 +56,7 @@ public class IntersectionUnionPagingIT extends AbstractIteratingQueryIT {
Set<String> created = performSetup( collectionIoHelper );
- testUnionPaging( collectionIoHelper, unionScan, created );
+// testUnionPaging( collectionIoHelper, unionScan, created );
testUnionPaging( collectionIoHelper, scanUnion, created );
}
@@ -76,7 +77,7 @@ public class IntersectionUnionPagingIT extends AbstractIteratingQueryIT {
private Set<String> performSetup( final IoHelper io ) throws Exception {
io.doSetup();
- int size = 500;
+ int size = 40;
long start = System.currentTimeMillis();
@@ -92,17 +93,16 @@ public class IntersectionUnionPagingIT extends AbstractIteratingQueryIT {
entity.put( "name", name );
entity.put( "fieldDate", "0000-00-00" );
- String field1;
+ String field1 = String.format( "%08d", i + 1 );
String field2;
//use a value slightly smaller than page size, since we want to simulate
//the cursor issues with union queries
- if ( i < PAGE_SIZE - 10 ) {
- field1 = String.format( "%08d", i + 1 );
+
+ if ( i < size - 10 ) {
field2 = zeros;
}
else {
- field1 = zeros;
field2 = String.format( "%08d", i + 1 );
}
@@ -111,7 +111,9 @@ public class IntersectionUnionPagingIT extends AbstractIteratingQueryIT {
entity.put( "field1Or", field1 );
entity.put( "field2Or", field2 );
- io.writeEntity( entity );
+ Entity saved = io.writeEntity( entity );
+
+ LOG.info("Writing entity with id '{}'", saved.getUuid());
}
long stop = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/16455b87/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java b/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java
index c165caa..9ad4623 100644
--- a/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java
+++ b/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java
@@ -49,11 +49,13 @@ public class IntersectionIteratorTest {
// we should get intersection on 1, 3, and 8
InOrderIterator first = new InOrderIterator( 100 );
+ first.add( id9 );
+ first.add( id8 );
first.add( id1 );
first.add( id2 );
first.add( id3 );
- first.add( id8 );
- first.add( id9 );
+
+
InOrderIterator second = new InOrderIterator( 100 );
second.add( id1 );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/16455b87/stack/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/log4j.properties b/stack/core/src/test/resources/log4j.properties
index 4c75404..9282771 100644
--- a/stack/core/src/test/resources/log4j.properties
+++ b/stack/core/src/test/resources/log4j.properties
@@ -18,7 +18,7 @@
# and the pattern to %c instead of %l. (%l is slower.)
# output messages into a rolling log file as well as stdout
-log4j.rootLogger=INFO,stdout
+log4j.rootLogger=DEBUG,stdout
# stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
[8/9] git commit: Fixes off by 1 error when using the start parameter
in collection traversal
Posted by sn...@apache.org.
Fixes off by 1 error when using the start parameter in collection traversal
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/acc95d13
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/acc95d13
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/acc95d13
Branch: refs/heads/master
Commit: acc95d131b33cc56dd0df3979bd75a40fc867678
Parents: b05edcd
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Jan 28 19:24:13 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jan 28 19:33:27 2014 -0700
----------------------------------------------------------------------
.../org/usergrid/persistence/cassandra/CassandraService.java | 5 +++--
.../org/usergrid/persistence/cassandra/RelationManagerImpl.java | 3 ++-
stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java | 2 +-
.../src/main/java/org/usergrid/tools/UniqueIndexCleanup.java | 2 +-
4 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/acc95d13/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java b/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java
index 7edc530..6e166b6 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java
@@ -1047,7 +1047,7 @@ public class CassandraService {
* @throws Exception the exception
*/
public IndexScanner getIdList( Keyspace ko, Object key, UUID start, UUID finish, int count, boolean reversed,
- IndexBucketLocator locator, UUID applicationId, String collectionName )
+ IndexBucketLocator locator, UUID applicationId, String collectionName, boolean keepFirst )
throws Exception {
if ( count <= 0 ) {
@@ -1058,7 +1058,8 @@ public class CassandraService {
start = null;
}
- final boolean skipFirst = start != null;
+
+ final boolean skipFirst = start != null && !keepFirst;
IndexScanner scanner =
new IndexBucketScanner( this, locator, ENTITY_ID_SETS, applicationId, IndexType.COLLECTION, key, start,
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/acc95d13/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java
index 7eb7693..577f803 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -2123,10 +2123,11 @@ public class RelationManagerImpl implements RelationManager {
startId = UUID_PARSER.parse( slice.getCursor() ).getUUID();
}
+
IndexScanner indexScanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ),
key( headEntity.getUuid(), DICTIONARY_COLLECTIONS, collectionName ), startId, null,
queryProcessor.getPageSizeHint( node ), query.isReversed(), indexBucketLocator, applicationId,
- collectionName );
+ collectionName, node.isForceKeepFirst() );
this.results.push( new SliceIterator( slice, indexScanner, UUID_PARSER ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/acc95d13/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java b/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java
index d886f70..f090a23 100644
--- a/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java
+++ b/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java
@@ -124,7 +124,7 @@ public class EntityCleanup extends ToolBase {
IndexScanner scanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ),
key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
- indexBucketLocator, applicationId, collectionName );
+ indexBucketLocator, applicationId, collectionName, false );
SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/acc95d13/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java
index f46a882..1b5dff7 100644
--- a/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java
@@ -179,7 +179,7 @@ public class UniqueIndexCleanup extends ToolBase {
IndexScanner scanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ),
key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
- indexBucketLocator, applicationId, collectionName );
+ indexBucketLocator, applicationId, collectionName, false );
SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser() );
[4/9] git commit: Fixes off by 1 error at root iterators
Posted by sn...@apache.org.
Fixes off by 1 error at root iterators
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8d4425d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8d4425d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8d4425d5
Branch: refs/heads/master
Commit: 8d4425d51ccf97d603dfeb0e587c432ad3c766a6
Parents: b7e3cf2
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Jan 28 18:33:09 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jan 28 18:36:11 2014 -0700
----------------------------------------------------------------------
.../persistence/cassandra/CassandraService.java | 4 +-
.../cassandra/RelationManagerImpl.java | 21 ++----
.../cassandra/index/ConnectedIndexScanner.java | 77 ++++++++++++++++----
.../cassandra/index/IndexBucketScanner.java | 33 +++++++--
.../persistence/query/ir/SearchVisitor.java | 7 +-
.../query/ir/result/SliceIterator.java | 15 +---
.../query/AllInConnectionNoTypeIT.java | 1 +
.../ir/result/IntersectionIteratorTest.java | 6 +-
.../java/org/usergrid/tools/EntityCleanup.java | 2 +-
.../org/usergrid/tools/UniqueIndexCleanup.java | 2 +-
10 files changed, 108 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d4425d5/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java b/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java
index a6b6268..7edc530 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/cassandra/CassandraService.java
@@ -1058,9 +1058,11 @@ public class CassandraService {
start = null;
}
+ final boolean skipFirst = start != null;
+
IndexScanner scanner =
new IndexBucketScanner( this, locator, ENTITY_ID_SETS, applicationId, IndexType.COLLECTION, key, start,
- finish, reversed, count, collectionName );
+ finish, reversed, count, skipFirst, collectionName );
return scanner;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d4425d5/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java
index 4f565c7..7eb7693 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -1379,7 +1379,7 @@ public class RelationManagerImpl implements RelationManager {
IndexScanner scanner =
new IndexBucketScanner( cass, indexBucketLocator, ENTITY_INDEX, applicationId, IndexType.CONNECTION,
- keyPrefix, range[0], range[1], slice.isReversed(), pageSize, slice.getPropertyName() );
+ keyPrefix, range[0], range[1], slice.isReversed(), pageSize, slice.hasCursor(), slice.getPropertyName() );
return scanner;
}
@@ -1400,14 +1400,9 @@ public class RelationManagerImpl implements RelationManager {
Object keyPrefix = key( indexKey, slice.getPropertyName() );
- // we have a cursor, so the first record should be discarded
- if ( slice.hasCursor() ) {
- pageSize++;
- }
-
IndexScanner scanner =
new IndexBucketScanner( cass, indexBucketLocator, ENTITY_INDEX, applicationId, IndexType.COLLECTION,
- keyPrefix, range[0], range[1], slice.isReversed(), pageSize, collectionName );
+ keyPrefix, range[0], range[1], slice.isReversed(), pageSize, slice.hasCursor(), collectionName );
return scanner;
}
@@ -2128,14 +2123,12 @@ public class RelationManagerImpl implements RelationManager {
startId = UUID_PARSER.parse( slice.getCursor() ).getUUID();
}
- boolean skipFirst = node.isForceKeepFirst() ? false : slice.hasCursor();
-
IndexScanner indexScanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ),
key( headEntity.getUuid(), DICTIONARY_COLLECTIONS, collectionName ), startId, null,
queryProcessor.getPageSizeHint( node ), query.isReversed(), indexBucketLocator, applicationId,
collectionName );
- this.results.push( new SliceIterator( slice, indexScanner, UUID_PARSER, skipFirst ) );
+ this.results.push( new SliceIterator( slice, indexScanner, UUID_PARSER ) );
}
@@ -2267,10 +2260,6 @@ public class RelationManagerImpl implements RelationManager {
start = slice.getCursor();
}
- // we'll discard the first match, increase the size
- if ( start != null ) {
- size++;
- }
boolean skipFirst = node.isForceKeepFirst() ? false : slice.hasCursor();
@@ -2316,9 +2305,9 @@ public class RelationManagerImpl implements RelationManager {
IndexScanner connectionScanner =
new ConnectedIndexScanner( cass, dictionaryType, applicationId, entityIdToUse, connectionTypes,
- start, slice.isReversed(), size );
+ start, slice.isReversed(), size, skipFirst );
- this.results.push( new SliceIterator( slice, connectionScanner, connectionParser, skipFirst ) );
+ this.results.push( new SliceIterator( slice, connectionScanner, connectionParser ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d4425d5/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java b/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
index 265dc4d..378f2bd 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/ConnectedIndexScanner.java
@@ -34,7 +34,9 @@ import static org.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COMPOSITE_
import static org.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
-/** @author tnine */
+/**
+ * @author tnine
+ */
public class ConnectedIndexScanner implements IndexScanner {
private final CassandraService cass;
@@ -44,24 +46,35 @@ public class ConnectedIndexScanner implements IndexScanner {
private final String dictionaryType;
private final UUID entityId;
private final Iterator<String> connectionTypes;
+ private final boolean skipFirst;
- /** Pointer to our next start read */
+
+ /**
+ * Pointer to our next start read
+ */
private ByteBuffer start;
- /** Set to the original value to start scanning from */
+ /**
+ * Set to the original value to start scanning from
+ */
private ByteBuffer scanStart;
- /** Iterator for our results from the last page load */
+ /**
+ * Iterator for our results from the last page load
+ */
private LinkedHashSet<HColumn<ByteBuffer, ByteBuffer>> lastResults;
- /** True if our last load loaded a full page size. */
+ /**
+ * True if our last load loaded a full page size.
+ */
private boolean hasMore = true;
private String currentConnectionType;
public ConnectedIndexScanner( CassandraService cass, String dictionaryType, UUID applicationId, UUID entityId,
- Iterator<String> connectionTypes, ByteBuffer start, boolean reversed, int pageSize ) {
+ Iterator<String> connectionTypes, ByteBuffer start, boolean reversed, int pageSize,
+ boolean skipFirst ) {
Assert.notNull( entityId, "Entity id for row key construction must be specified when searching graph indexes" );
// create our start and end ranges
@@ -74,6 +87,7 @@ public class ConnectedIndexScanner implements IndexScanner {
this.pageSize = pageSize;
this.dictionaryType = dictionaryType;
this.connectionTypes = connectionTypes;
+ this.skipFirst = skipFirst;
if ( connectionTypes.hasNext() ) {
@@ -106,14 +120,32 @@ public class ConnectedIndexScanner implements IndexScanner {
return false;
}
+ boolean skipFirst = this.skipFirst && start == scanStart;
+
+ int totalSelectSize = pageSize + 1;
+
+ //we're discarding the first, so increase our total size by 1 since this value will be inclusive in the seek
+ if ( skipFirst ) {
+ totalSelectSize++;
+ }
+
lastResults = new LinkedHashSet<HColumn<ByteBuffer, ByteBuffer>>();
+
+ //cleanup columns for later logic
+ //pointer to the first col we load
+ HColumn<ByteBuffer, ByteBuffer> first = null;
+
+ //pointer to the last column we load
+ HColumn<ByteBuffer, ByteBuffer> last = null;
+
//go through each connection type until we exhaust the result sets
while ( currentConnectionType != null ) {
//only load a delta size to get this next page
- int selectSize = pageSize + 1 - lastResults.size();
+ int selectSize = totalSelectSize - lastResults.size();
+
Object key = key( entityId, dictionaryType, currentConnectionType );
@@ -122,17 +154,24 @@ public class ConnectedIndexScanner implements IndexScanner {
cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_COMPOSITE_DICTIONARIES, key,
start, null, selectSize, reversed );
+ final int resultSize = results.size();
+
+ if(resultSize > 0){
+
+ last = results.get( resultSize -1 );
+
+ if(first == null ){
+ first = results.get( 0 );
+ }
+ }
+
lastResults.addAll( results );
+
// we loaded a full page, there might be more
- if ( results.size() == selectSize ) {
+ if ( resultSize == selectSize ) {
hasMore = true;
- // set the bytebuffer for the next pass
- start = results.get( results.size() - 1 ).getName();
-
- lastResults.remove( lastResults.size() - 1 );
-
//we've loaded a full page
break;
}
@@ -152,6 +191,16 @@ public class ConnectedIndexScanner implements IndexScanner {
}
}
+ //remove the first element, we need to skip it
+ if ( skipFirst && first != null) {
+ lastResults.remove( first );
+ }
+
+ if ( hasMore && last != null ) {
+ // set the bytebuffer for the next pass
+ start = last.getName();
+ lastResults.remove( last );
+ }
return lastResults != null && lastResults.size() > 0;
}
@@ -199,7 +248,7 @@ public class ConnectedIndexScanner implements IndexScanner {
* @see java.util.Iterator#next()
*/
@Override
- @Metered(group = "core", name = "IndexBucketScanner_load")
+ @Metered( group = "core", name = "IndexBucketScanner_load" )
public Set<HColumn<ByteBuffer, ByteBuffer>> next() {
Set<HColumn<ByteBuffer, ByteBuffer>> returnVal = lastResults;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d4425d5/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/IndexBucketScanner.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/IndexBucketScanner.java b/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/IndexBucketScanner.java
index 1a0776a..5c234d9 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/IndexBucketScanner.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/cassandra/index/IndexBucketScanner.java
@@ -55,6 +55,7 @@ public class IndexBucketScanner implements IndexScanner {
private final int pageSize;
private final String[] indexPath;
private final IndexType indexType;
+ private final boolean skipFirst;
/** Pointer to our next start read */
private Object start;
@@ -69,9 +70,10 @@ public class IndexBucketScanner implements IndexScanner {
private boolean hasMore = true;
+
public IndexBucketScanner( CassandraService cass, IndexBucketLocator locator, ApplicationCF columnFamily,
UUID applicationId, IndexType indexType, Object keyPrefix, Object start, Object finish,
- boolean reversed, int pageSize, String... indexPath ) {
+ boolean reversed, int pageSize, boolean skipFirst, String... indexPath) {
this.cass = cass;
this.indexBucketLocator = locator;
this.applicationId = applicationId;
@@ -80,7 +82,10 @@ public class IndexBucketScanner implements IndexScanner {
this.start = start;
this.finish = finish;
this.reversed = reversed;
- this.pageSize = pageSize;
+ this.skipFirst = skipFirst;
+
+ //we always add 1 to the page size. This is because we pop the last column for the next page of results
+ this.pageSize = pageSize+1;
this.indexPath = indexPath;
this.indexType = indexType;
this.scanStart = start;
@@ -121,24 +126,40 @@ public class IndexBucketScanner implements IndexScanner {
//if we skip the first we need to set the load to page size +2, since we'll discard the first
//and start paging at the next entity, otherwise we'll just load the page size we need
- int selectSize = pageSize + 1;
+ int selectSize = pageSize;
+
+ //we purposefully use instance equality. If it's a pointer to the same value, we need to increase by 1
+ //since we'll be skipping the first value
+
+ final boolean firstPageSkipFirst = this.skipFirst && start == scanStart;
+
+ if(firstPageSkipFirst){
+ selectSize++;
+ }
TreeSet<HColumn<ByteBuffer, ByteBuffer>> resultsTree = IndexMultiBucketSetLoader
.load( cass, columnFamily, applicationId, cassKeys, start, finish, selectSize, reversed );
+ //remove the first element, it's from a cursor value and we don't want to retain it
+
+
// we loaded a full page, there might be more
if ( resultsTree.size() == selectSize ) {
hasMore = true;
- // set the bytebuffer for the next pass
- start = resultsTree.last().getName();
- resultsTree.remove( resultsTree.last() );
+ // set the bytebuffer for the next pass
+ start = resultsTree.pollLast().getName();
}
else {
hasMore = false;
}
+ //remove the first element since it needs to be skipped AFTER the size check. Otherwise it will fail
+ if ( firstPageSkipFirst ) {
+ resultsTree.pollFirst();
+ }
+
lastResults = resultsTree;
return lastResults != null && lastResults.size() > 0;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d4425d5/stack/core/src/main/java/org/usergrid/persistence/query/ir/SearchVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/SearchVisitor.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/SearchVisitor.java
index 665ddd2..02e8386 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/SearchVisitor.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/SearchVisitor.java
@@ -170,8 +170,7 @@ public abstract class SearchVisitor implements NodeVisitor {
if ( subResults == null ) {
QuerySlice firstFieldSlice = new QuerySlice( slice.getPropertyName(), -1 );
subResults =
- new SliceIterator( slice, secondaryIndexScan( orderByNode, firstFieldSlice ), COLLECTION_PARSER,
- slice.hasCursor() );
+ new SliceIterator( slice, secondaryIndexScan( orderByNode, firstFieldSlice ), COLLECTION_PARSER );
}
orderIterator = new OrderByIterator( slice, orderByNode.getSecondarySorts(), subResults, em,
@@ -190,7 +189,7 @@ public abstract class SearchVisitor implements NodeVisitor {
scanner = secondaryIndexScan( orderByNode, slice );
}
- SliceIterator joinSlice = new SliceIterator( slice, scanner, COLLECTION_PARSER, slice.hasCursor() );
+ SliceIterator joinSlice = new SliceIterator( slice, scanner, COLLECTION_PARSER);
IntersectionIterator union = new IntersectionIterator( queryProcessor.getPageSizeHint( orderByNode ) );
union.addIterator( joinSlice );
@@ -221,7 +220,7 @@ public abstract class SearchVisitor implements NodeVisitor {
for ( QuerySlice slice : node.getAllSlices() ) {
IndexScanner scanner = secondaryIndexScan( node, slice );
- intersections.addIterator( new SliceIterator( slice, scanner, COLLECTION_PARSER, slice.hasCursor() ) );
+ intersections.addIterator( new SliceIterator( slice, scanner, COLLECTION_PARSER) );
}
results.push( intersections );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d4425d5/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
index 6922b9f..82f60c2 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
@@ -47,7 +47,6 @@ public class SliceIterator implements ResultIterator {
private final SliceParser parser;
private final IndexScanner scanner;
private final int pageSize;
- private final boolean skipFirst;
/**
* Pointer to the uuid set until it's returned
@@ -74,13 +73,11 @@ public class SliceIterator implements ResultIterator {
* @param scanner The scanner to use to read the cols
* @param slice The slice used in the scanner
* @param parser The parser for the scanner results
- * @param skipFirst True if the first record should be skipped, used with cursors
*/
- public SliceIterator( QuerySlice slice, IndexScanner scanner, SliceParser parser, boolean skipFirst ) {
+ public SliceIterator( QuerySlice slice, IndexScanner scanner, SliceParser parser ) {
this.slice = slice;
this.parser = parser;
this.scanner = scanner;
- this.skipFirst = skipFirst;
this.pageSize = scanner.getPageSize();
this.cols = new LinkedHashMap<UUID, ScanColumn>( this.pageSize );
this.parsedCols = new LinkedHashSet<ScanColumn>( this.pageSize );
@@ -122,13 +119,6 @@ public class SliceIterator implements ResultIterator {
cols.clear();
- /**
- * Skip the first value, it's from the previous cursor
- */
- if ( skipFirst && results.hasNext() ) {
- results.next();
- }
-
parsedCols.clear();
while ( results.hasNext() ) {
@@ -142,9 +132,6 @@ public class SliceIterator implements ResultIterator {
continue;
}
-
- logger.debug( "Parsed column with uuid '{}'", parsed.getUUID() );
-
last = parsed;
cols.put( parsed.getUUID(), parsed );
parsedCols.add( parsed );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d4425d5/stack/core/src/test/java/org/usergrid/persistence/query/AllInConnectionNoTypeIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/usergrid/persistence/query/AllInConnectionNoTypeIT.java b/stack/core/src/test/java/org/usergrid/persistence/query/AllInConnectionNoTypeIT.java
index 07d4c46..34f8cfe 100644
--- a/stack/core/src/test/java/org/usergrid/persistence/query/AllInConnectionNoTypeIT.java
+++ b/stack/core/src/test/java/org/usergrid/persistence/query/AllInConnectionNoTypeIT.java
@@ -24,6 +24,7 @@ import org.usergrid.persistence.Results;
/** @author tnine */
public class AllInConnectionNoTypeIT extends AbstractIteratingQueryIT {
+
@Test
public void allInConnectionNoType() throws Exception {
allIn( new ConnectionNoTypeHelper(app) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d4425d5/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java b/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java
index 9ad4623..ac8c2cb 100644
--- a/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java
+++ b/stack/core/src/test/java/org/usergrid/persistence/query/ir/result/IntersectionIteratorTest.java
@@ -91,13 +91,13 @@ public class IntersectionIteratorTest {
// now make sure it's right, only 1, 3 and 8 intersect
assertTrue( union.hasNext() );
- assertEquals( id1, union.next().getUUID() );
+ assertEquals( id8, union.next().getUUID() );
assertTrue( union.hasNext() );
- assertEquals( id3, union.next().getUUID() );
+ assertEquals( id1, union.next().getUUID() );
assertTrue( union.hasNext() );
- assertEquals( id8, union.next().getUUID() );
+ assertEquals( id3, union.next().getUUID() );
assertFalse( union.hasNext() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d4425d5/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java b/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java
index ba79980..d886f70 100644
--- a/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java
+++ b/stack/tools/src/main/java/org/usergrid/tools/EntityCleanup.java
@@ -126,7 +126,7 @@ public class EntityCleanup extends ToolBase {
key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
indexBucketLocator, applicationId, collectionName );
- SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser(), false );
+ SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser() );
while ( itr.hasNext() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d4425d5/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java
index 367d1da..f46a882 100644
--- a/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/usergrid/tools/UniqueIndexCleanup.java
@@ -181,7 +181,7 @@ public class UniqueIndexCleanup extends ToolBase {
key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
indexBucketLocator, applicationId, collectionName );
- SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser(), false );
+ SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser() );
while ( itr.hasNext() ) {
[6/9] git commit: Refactored check next logic
Posted by sn...@apache.org.
Refactored check next logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b7e3cf29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b7e3cf29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b7e3cf29
Branch: refs/heads/master
Commit: b7e3cf290f162221bf96a01a5f8e115ce59b33d9
Parents: 2303624
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Jan 27 10:11:56 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jan 28 18:36:11 2014 -0700
----------------------------------------------------------------------
.../persistence/query/ir/result/MergeIterator.java | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b7e3cf29/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java
index e09cff5..81eacd8 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/MergeIterator.java
@@ -62,7 +62,7 @@ public abstract class MergeIterator implements ResultIterator {
@Override
public boolean hasNext() {
//if next isn't set, try to advance
- if(next != null){
+ if(checkNext()){
return true;
}
@@ -70,7 +70,16 @@ public abstract class MergeIterator implements ResultIterator {
doAdvance();
- return next != null;
+ return checkNext();
+ }
+
+
+ /**
+ * Single source of logic to check if a next is present.
+ * @return
+ */
+ protected boolean checkNext(){
+ return next != null && next.size() > 0;
}
[2/9] git commit: Fixed off by 1 issue with loading too many pages.
Posted by sn...@apache.org.
Fixed off by 1 issue with loading too many pages.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ba1f456e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ba1f456e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ba1f456e
Branch: refs/heads/master
Commit: ba1f456ef13f5cd44595c185e20a32b7eaead279
Parents: b5492f9
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Jan 21 13:45:31 2014 -0800
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jan 21 13:45:31 2014 -0800
----------------------------------------------------------------------
.../org/usergrid/persistence/query/ir/result/SliceIterator.java | 2 +-
.../org/usergrid/persistence/query/ir/result/UnionIterator.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ba1f456e/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
index 6cc833d..855a844 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/SliceIterator.java
@@ -125,7 +125,7 @@ public class SliceIterator implements ResultIterator {
/**
* Skip the first value, it's from the previous cursor
*/
- if ( skipFirst && pagesLoaded == 0 && results.hasNext() ) {
+ if ( skipFirst && results.hasNext() ) {
results.next();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ba1f456e/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java
index 00e90b8..c7b31d1 100644
--- a/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java
+++ b/stack/core/src/main/java/org/usergrid/persistence/query/ir/result/UnionIterator.java
@@ -165,7 +165,7 @@ public class UnionIterator extends MultiIterator {
index = ( index * -1 ) - 1;
- //outside the renage
+ //outside the range
if ( index >= maxSize ) {
return;
}
[9/9] git commit: Merge pull request #41 from
usergrid/USERGRID-2862-limitfix
Posted by sn...@apache.org.
Merge pull request #41 from usergrid/USERGRID-2862-limitfix
Usergrid 2862 limitfix
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c36f5ae6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c36f5ae6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c36f5ae6
Branch: refs/heads/master
Commit: c36f5ae6811f579ce22941b409b15ef41efb83fa
Parents: 391e7f4 acc95d1
Author: tonuquq <to...@gmail.com>
Authored: Wed Jan 29 08:09:44 2014 -0800
Committer: tonuquq <to...@gmail.com>
Committed: Wed Jan 29 08:09:44 2014 -0800
----------------------------------------------------------------------
.../persistence/cassandra/CassandraService.java | 7 +-
.../persistence/cassandra/QueryProcessor.java | 2 +-
.../cassandra/RelationManagerImpl.java | 22 ++---
.../cassandra/index/ConnectedIndexScanner.java | 77 +++++++++++++----
.../cassandra/index/IndexBucketScanner.java | 33 ++++++--
.../persistence/query/ir/SearchVisitor.java | 7 +-
.../query/ir/result/IntersectionIterator.java | 4 +-
.../query/ir/result/MergeIterator.java | 28 +++++--
.../query/ir/result/SliceIterator.java | 12 +--
.../query/ir/result/UnionIterator.java | 17 +++-
.../query/AllInConnectionNoTypeIT.java | 1 +
.../query/IntersectionUnionPagingIT.java | 20 +++--
.../ir/result/IntersectionIteratorTest.java | 12 +--
.../query/ir/result/UnionIteratorTest.java | 88 ++++++++++++++++++++
stack/tools/pom.xml | 4 +
.../java/org/usergrid/tools/EntityCleanup.java | 4 +-
.../org/usergrid/tools/UniqueIndexCleanup.java | 4 +-
.../java/org/usergrid/tools/bean/ExportOrg.java | 11 +++
18 files changed, 272 insertions(+), 81 deletions(-)
----------------------------------------------------------------------