You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/07/30 20:44:38 UTC
[15/35] incubator-usergrid git commit: Fixes multi order by
Fixes multi order by
Cleans up unused code and adds license headers
Fixes export tools.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/44448037
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/44448037
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/44448037
Branch: refs/heads/ug2-doc-update
Commit: 4444803727620198045fd06143255495227bac4a
Parents: 501da72
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Jul 6 16:43:05 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Jul 6 16:43:05 2015 -0600
----------------------------------------------------------------------
.../query/ir/result/AbstractScanColumn.java | 15 +-
.../query/ir/result/GatherIterator.java | 16 ++
.../query/ir/result/GeoIterator.java | 20 --
.../query/ir/result/OrderByIterator.java | 144 ++++++++------
.../persistence/query/ir/result/ScanColumn.java | 17 --
.../ir/result/SearchCollectionVisitor.java | 16 ++
.../ir/result/SearchConnectionVisitor.java | 16 ++
.../query/ir/result/SliceCursorGenerator.java | 4 +-
.../persistence/query/ir/result/UUIDColumn.java | 16 ++
.../query/ir/result/UnionIterator.java | 17 --
.../apache/usergrid/tools/EntityCleanup.java | 80 ++++----
.../usergrid/tools/EntityInsertBenchMark.java | 3 +-
.../usergrid/tools/EntityReadBenchMark.java | 11 +-
.../usergrid/tools/UniqueIndexCleanup.java | 186 ++++++++++---------
14 files changed, 300 insertions(+), 261 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
index 35672d9..cabb710 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
@@ -52,7 +52,7 @@ public abstract class AbstractScanColumn implements ScanColumn {
}
- @Override
+
public ByteBuffer getCursorValue() {
return buffer == null ? null : buffer.duplicate();
}
@@ -87,19 +87,6 @@ public abstract class AbstractScanColumn implements ScanColumn {
'}';
}
-
- @Override
- public void setChild( final ScanColumn childColumn ) {
- this.child = childColumn;
- }
-
-
- @Override
- public ScanColumn getChild() {
- return child;
- }
-
-
@Override
public void addToCursor( final CursorCache cache ) {
this.sliceCursorGenerator.addToCursor( cache, this );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
index 9d2e4da..e199d33 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GatherIterator.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.usergrid.persistence.query.ir.result;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
index 6b5f527..7f09bb3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/GeoIterator.java
@@ -259,9 +259,6 @@ public class GeoIterator implements ResultIterator {
private final EntityLocationRef location;
private final GeoCursorGenerator geoCursorGenerator;
- private ScanColumn child;
-
-
public LocationScanColumn( EntityLocationRef location, final GeoCursorGenerator geoCursorGenerator ) {
this.location = location;
@@ -275,18 +272,6 @@ public class GeoIterator implements ResultIterator {
}
- @Override
- public ByteBuffer getCursorValue() {
- throw new UnsupportedOperationException(
- "This is not supported for location scan columns. It requires iterator information" );
- }
-
-
- @Override
- public void setChild( final ScanColumn childColumn ) {
- this.child = childColumn;
- }
-
@Override
public void addToCursor( final CursorCache cache ) {
@@ -294,11 +279,6 @@ public class GeoIterator implements ResultIterator {
}
- @Override
- public ScanColumn getChild() {
- return this.child;
- }
-
@Override
public boolean equals( Object o ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
index 6d05bd6..13fd306 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/OrderByIterator.java
@@ -21,17 +21,17 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import org.apache.commons.collections.comparators.ComparatorChain;
+
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityPropertyComparator;
@@ -40,9 +40,8 @@ import org.apache.usergrid.persistence.Query.SortPredicate;
import org.apache.usergrid.persistence.cassandra.CursorCache;
import org.apache.usergrid.persistence.query.ir.QuerySlice;
-import org.apache.commons.collections.comparators.ComparatorChain;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
/**
* 1) Take a result set iterator as the child 2) Iterate only over candidates and create a cursor from the candidates
@@ -63,6 +62,8 @@ public class OrderByIterator extends MergeIterator {
//our last result from in memory sorting
private SortedEntitySet entries;
+ private final UUIDCursorGenerator<MultiColumnSort> generator;
+
/**
* @param pageSize
@@ -91,6 +92,8 @@ public class OrderByIterator extends MergeIterator {
// paging
this.secondaryFields.add( NAME_UUID );
this.subSortCompare.addComparator( new EntityPropertyComparator( NAME_UUID, false ) );
+
+ this.generator = new UUIDCursorGenerator<MultiColumnSort>( this.slice.hashCode() );
}
@@ -105,7 +108,7 @@ public class OrderByIterator extends MergeIterator {
minEntryId = ue.fromByteBuffer( cursor );
}
- entries = new SortedEntitySet( subSortCompare, em, secondaryFields, pageSize, minEntryId );
+ entries = new SortedEntitySet( subSortCompare, em, secondaryFields, pageSize, minEntryId, generator );
/**
* keep looping through our peek iterator. We need to inspect each forward page to ensure we have performed a
@@ -134,45 +137,75 @@ public class OrderByIterator extends MergeIterator {
// no op
}
-//
-// @Override
-// public void finalizeCursor( CursorCache cache, UUID lastValue ) {
-// int sliceHash = slice.hashCode();
-//
-// ByteBuffer bytes = ue.toByteBuffer( lastValue );
-//
-// if ( bytes == null ) {
-// return;
-// }
-//
-// cache.setNextCursor( sliceHash, bytes );
-// }
+ //
+ // @Override
+ // public void finalizeCursor( CursorCache cache, UUID lastValue ) {
+ // int sliceHash = slice.hashCode();
+ //
+ // ByteBuffer bytes = ue.toByteBuffer( lastValue );
+ //
+ // if ( bytes == null ) {
+ // return;
+ // }
+ //
+ // cache.setNextCursor( sliceHash, bytes );
+ // }
/** A Sorted set with a max size. When a new entry is added, the max is removed */
- public static final class SortedEntitySet extends TreeSet<Entity> {
+ public static final class SortedEntitySet {
private final int maxSize;
- private final Map<UUID, ScanColumn> cursorVal = new HashMap<UUID, ScanColumn>();
+ private final UUIDCursorGenerator<MultiColumnSort> generator;
+ private final Set<UUID> uuidBuffer = new LinkedHashSet<UUID>();
+ private final TreeSet<ScanColumn> sortedEntities = new TreeSet<ScanColumn>();
private final EntityManager em;
private final List<String> fields;
private final Entity minEntity;
private final Comparator<Entity> comparator;
- public SortedEntitySet( Comparator<Entity> comparator, EntityManager em, List<String> fields, int maxSize,
- UUID minEntityId ) {
- super( comparator );
+ public SortedEntitySet( final Comparator<Entity> comparator, final EntityManager em, final List<String> fields,
+ final int maxSize, final UUID minEntityId,
+ final UUIDCursorGenerator<MultiColumnSort> generator ) {
this.maxSize = maxSize;
this.em = em;
this.fields = fields;
this.comparator = comparator;
+ this.generator = generator;
this.minEntity = getPartialEntity( minEntityId );
}
- @Override
- public boolean add( Entity entity ) {
+ /** Add the current scancolumn to be loaded **/
+ public void add( ScanColumn col ) {
+ uuidBuffer.add( col.getUUID() );
+ }
+
+
+ public void load() {
+ try {
+
+
+ //load all the entities
+ for ( Entity e : em.getPartialEntities( uuidBuffer, fields ) ) {
+ add( e );
+ }
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to load partial entities", e );
+ throw new RuntimeException( e );
+ }
+ }
+
+
+ /** Turn our sorted entities into a set of ids */
+ public Set<ScanColumn> toIds() {
+ return sortedEntities;
+ }
+
+
+ private boolean add( Entity entity ) {
// don't add this entity. We get it in our scan range, but it's <= the minimum value that
//should be allowed in the result set
@@ -180,25 +213,17 @@ public class OrderByIterator extends MergeIterator {
return false;
}
- boolean added = super.add( entity );
+ boolean added = sortedEntities.add( new MultiColumnSort( entity, comparator, generator ) );
- while ( size() > maxSize ) {
+ while ( sortedEntities.size() > maxSize ) {
//remove our last element, we're over size
- Entity e = this.pollLast();
- //remove it from the cursors as well
- cursorVal.remove( e.getUuid() );
+ sortedEntities.pollLast();
}
return added;
}
- /** add the id to be loaded, and the dynamiccomposite column that belongs with it */
- public void add( ScanColumn col ) {
- cursorVal.put( col.getUUID(), col );
- }
-
-
private Entity getPartialEntity( UUID minEntityId ) {
List<Entity> entities;
@@ -216,35 +241,42 @@ public class OrderByIterator extends MergeIterator {
return entities.get( 0 );
}
+ }
- public void load() {
- try {
- for ( Entity e : em.getPartialEntities( cursorVal.keySet(), fields ) ) {
- add( e );
- }
- }
- catch ( Exception e ) {
- logger.error( "Unable to load partial entities", e );
- throw new RuntimeException( e );
- }
+ private static final class MultiColumnSort implements ScanColumn {
+
+ private final CursorGenerator<MultiColumnSort> generator;
+ private final Entity partialCompareEntity;
+ private final Comparator<Entity> entityComparator;
+
+
+ private MultiColumnSort( final Entity partialCompareEntity, final Comparator<Entity> entityComparator,
+ final CursorGenerator<MultiColumnSort> generator ) {
+ this.generator = generator;
+ this.partialCompareEntity = partialCompareEntity;
+ this.entityComparator = entityComparator;
}
- /** Turn our sorted entities into a set of ids */
- public Set<ScanColumn> toIds() {
- Iterator<Entity> itr = iterator();
+ @Override
+ public UUID getUUID() {
+ return partialCompareEntity.getUuid();
+ }
- Set<ScanColumn> columns = new LinkedHashSet<ScanColumn>( this.size() );
- while ( itr.hasNext() ) {
+ @Override
+ public void addToCursor( final CursorCache cache ) {
+ this.generator.addToCursor( cache, this );
+ }
- UUID id = itr.next().getUuid();
- columns.add( cursorVal.get( id ) );
- }
+ @Override
+ public int compareTo( final ScanColumn o ) {
+
+ final MultiColumnSort other = ( MultiColumnSort ) o;
- return columns;
+ return entityComparator.compare( this.partialCompareEntity, other.partialCompareEntity );
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
index 577d34a..eaed2d3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/ScanColumn.java
@@ -30,17 +30,6 @@ public interface ScanColumn extends Comparable<ScanColumn> {
/** Get the uuid from the column */
UUID getUUID();
- /** Get the cursor value of this column */
- ByteBuffer getCursorValue();
-
- /**
- * Append the child column used in tree iterator to this column, along with the comparator used to compare them
- *
- * for instance, a term search of A = 1 AND B = 2 would generate a ScanColumn of A-- child -> B
- * @param childColumn
- */
- void setChild( final ScanColumn childColumn );
-
/**
* Use the generator to add this value to the cursor cache
@@ -49,10 +38,4 @@ public interface ScanColumn extends Comparable<ScanColumn> {
void addToCursor( final CursorCache cache );
- /**
- * Returns the childl column if present, can return null
- * @return
- */
- ScanColumn getChild();
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
index 32c3a6e..c991783 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.usergrid.persistence.query.ir.result;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index 5949ee4..f518297 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.usergrid.persistence.query.ir.result;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
index cad0a0f..d3fd534 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SliceCursorGenerator.java
@@ -29,7 +29,7 @@ import org.apache.usergrid.persistence.query.ir.QuerySlice;
/**
* A cursor generator for the specified query slice
*/
-public class SliceCursorGenerator implements CursorGenerator<ScanColumn> {
+public class SliceCursorGenerator implements CursorGenerator<AbstractScanColumn> {
private final QuerySlice slice;
@@ -38,7 +38,7 @@ public class SliceCursorGenerator implements CursorGenerator<ScanColumn> {
@Override
- public void addToCursor( final CursorCache cache, final ScanColumn col ) {
+ public void addToCursor( final CursorCache cache, final AbstractScanColumn col ) {
if ( col == null ) {
return;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
index 264a9b6..f5d0091 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UUIDColumn.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.usergrid.persistence.query.ir.result;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
index 58e0e9a..526fced 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/UnionIterator.java
@@ -253,28 +253,11 @@ public class UnionIterator extends MultiIterator {
@Override
- public ByteBuffer getCursorValue() {
- return ue.toByteBuffer( delegate.getUUID() );
- }
-
-
- @Override
- public void setChild( final ScanColumn childColumn ) {
- //intentionally a no-op, since child is on the delegate
- }
-
-
- @Override
public void addToCursor( final CursorCache cache ) {
this.uuidCursorGenerator.addToCursor( cache, this );
}
- @Override
- public ScanColumn getChild() {
- return delegate.getChild();
- }
-
@Override
public boolean equals( final Object o ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
index 368119c..3691031 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityCleanup.java
@@ -25,9 +25,14 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.cassandra.CassandraService;
@@ -38,11 +43,6 @@ import org.apache.usergrid.persistence.query.ir.result.ScanColumnTransformer;
import org.apache.usergrid.persistence.query.ir.result.SliceIterator;
import org.apache.usergrid.persistence.query.ir.result.UUIDIndexSliceParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.mutation.Mutator;
@@ -51,9 +51,9 @@ import static org.apache.usergrid.persistence.Schema.DICTIONARY_COLLECTIONS;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_ID_SETS;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
/**
@@ -122,53 +122,57 @@ public class EntityCleanup extends ToolBase {
// go through each collection and audit the value
for ( String collectionName : collectionNames ) {
- IndexScanner scanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ),
- key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
- indexBucketLocator, applicationId, collectionName, false );
- SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser( uuidCursorGenerator ) );
+ for ( final String bucketName : indexBucketLocator.getBuckets() ) {
- while ( itr.hasNext() ) {
+ IndexScanner scanner =
+ cass.getIdList( key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null,
+ PAGE_SIZE, false, bucketName, applicationId, false );
- // load all entity ids from the index itself.
+ SliceIterator itr = new SliceIterator( scanner, new UUIDIndexSliceParser( null ) );
- Set<ScanColumn> copy = new LinkedHashSet<ScanColumn>( itr.next() );
+ while ( itr.hasNext() ) {
- results = em.get( ScanColumnTransformer.getIds( copy ) );
- // nothing to do they're the same size so there's no
- // orphaned uuid's in the entity index
- if ( copy.size() == results.size() ) {
- continue;
- }
+ // load all entity ids from the index itself.
- // they're not the same, we have some orphaned records,
- // remove them
+ Set<ScanColumn> copy = new LinkedHashSet<ScanColumn>( itr.next() );
- for ( Entity returned : results.getEntities() ) {
- copy.remove( returned.getUuid() );
- }
+ results = em.get( ScanColumnTransformer.getIds( copy ) );
+ // nothing to do they're the same size so there's no
+ // orphaned uuid's in the entity index
+ if ( copy.size() == results.size() ) {
+ continue;
+ }
- // what's left needs deleted, do so
+ // they're not the same, we have some orphaned records,
+ // remove them
- logger.info( "Cleaning up {} orphaned entities for app {}", copy.size(), app.getValue() );
+ for ( Entity returned : results.getEntities() ) {
+ copy.remove( returned.getUuid() );
+ }
- Keyspace ko = cass.getApplicationKeyspace( applicationId );
- Mutator<ByteBuffer> m = createMutator( ko, be );
+ // what's left needs deleted, do so
- for ( ScanColumn col : copy ) {
+ logger.info( "Cleaning up {} orphaned entities for app {}", copy.size(), app.getValue() );
- final UUID id = col.getUUID();
+ Keyspace ko = cass.getApplicationKeyspace( applicationId );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
- Object collections_key = key( applicationId, Schema.DICTIONARY_COLLECTIONS, collectionName,
- indexBucketLocator
- .getBucket( applicationId, IndexType.COLLECTION, id, collectionName ) );
+ for ( ScanColumn col : copy ) {
- addDeleteToMutator( m, ENTITY_ID_SETS, collections_key, id, timestamp );
+ final UUID id = col.getUUID();
- logger.info( "Deleting entity with id '{}' from collection '{}'", id, collectionName );
- }
+ Object collections_key = key( applicationId, Schema.DICTIONARY_COLLECTIONS, collectionName,
+ indexBucketLocator
+ .getBucket( id ) );
+
+ addDeleteToMutator( m, ENTITY_ID_SETS, collections_key, id, timestamp );
- m.execute();
+ logger.info( "Deleting entity with id '{}' from collection '{}'", id, collectionName );
+ }
+
+ m.execute();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java
index fe1edbf..bf3d5d4 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityInsertBenchMark.java
@@ -29,7 +29,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.DynamicEntity;
import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
import org.apache.usergrid.utils.UUIDUtils;
@@ -173,7 +172,7 @@ public class EntityInsertBenchMark extends ToolBase {
String bucketId =
- indexBucketLocator.getBucket( appId, IndexType.COLLECTION, dynEntity.getUuid(), "test" );
+ indexBucketLocator.getBucket( dynEntity.getUuid());
Object index_name = key( appId, "tests", "test", bucketId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java
index 1b54495..83f8df4 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/EntityReadBenchMark.java
@@ -31,15 +31,15 @@ import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
-import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
-import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+
import com.yammer.metrics.Metrics;
import com.yammer.metrics.core.MetricPredicate;
import com.yammer.metrics.core.Timer;
@@ -60,8 +60,9 @@ import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_IND
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
import static org.apache.usergrid.persistence.cassandra.IndexUpdate.indexValueCode;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.dce;
import static org.apache.usergrid.utils.ConversionUtils.bytebuffers;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
/**
@@ -262,7 +263,7 @@ public class EntityReadBenchMark extends ToolBase {
private boolean read( String value ) {
- List<String> buckets = indexBucketLocator.getBuckets( appId, IndexType.UNIQUE, "tests" );
+ List<String> buckets = indexBucketLocator.getBuckets( );
List<Object> cassKeys = new ArrayList<Object>( buckets.size() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44448037/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index 6b669b9..994e850 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -29,13 +29,18 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
import org.apache.usergrid.management.ApplicationInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.Identifier;
import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
@@ -45,11 +50,6 @@ import org.apache.usergrid.persistence.query.ir.result.SliceIterator;
import org.apache.usergrid.persistence.query.ir.result.UUIDIndexSliceParser;
import org.apache.usergrid.persistence.schema.CollectionInfo;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.AbstractComposite.ComponentEquality;
import me.prettyprint.hector.api.beans.DynamicComposite;
@@ -64,10 +64,10 @@ import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_IND
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
import static org.apache.usergrid.persistence.cassandra.CassandraService.INDEX_ENTRY_LIST_COUNT;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.utils.CompositeUtils.setEqualityFlag;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
/**
@@ -177,123 +177,129 @@ public class UniqueIndexCleanup extends ToolBase {
for ( String collectionName : getCollectionNames( em, line ) ) {
- IndexScanner scanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ),
- key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
- indexBucketLocator, applicationId, collectionName, false );
+ for ( final String bucketName : indexBucketLocator.getBuckets() ) {
- SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser( uuidCursorGenerator ) );
+ IndexScanner scanner =
+ cass.getIdList( key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null,
+ PAGE_SIZE, false, bucketName, applicationId, false );
+ SliceIterator itr = new SliceIterator( scanner, new UUIDIndexSliceParser( null ) );
- while ( itr.hasNext() ) {
- Set<ScanColumn> ids = itr.next();
+ while ( itr.hasNext() ) {
- CollectionInfo collection = getDefaultSchema().getCollection( "application", collectionName );
+ Set<ScanColumn> ids = itr.next();
+ CollectionInfo collection = getDefaultSchema().getCollection( "application", collectionName );
- //We shouldn't have to do this, but otherwise the cursor won't work
- Set<String> indexed = collection.getPropertiesIndexed();
- // what's left needs deleted, do so
+ //We shouldn't have to do this, but otherwise the cursor won't work
+ Set<String> indexed = collection.getPropertiesIndexed();
- logger.info( "Auditing {} entities for collection {} in app {}", new Object[] {
- ids.size(), collectionName, app.getValue()
- } );
+ // what's left needs deleted, do so
- for ( ScanColumn col : ids ) {
- final UUID id = col.getUUID();
- boolean reIndex = false;
+ logger.info( "Auditing {} entities for collection {} in app {}", new Object[] {
+ ids.size(), collectionName, app.getValue()
+ } );
- Mutator<ByteBuffer> m = createMutator( ko, be );
+ for ( ScanColumn col : ids ) {
+ final UUID id = col.getUUID();
+ boolean reIndex = false;
- try {
+ Mutator<ByteBuffer> m = createMutator( ko, be );
- for ( String prop : indexed ) {
+ try {
- String bucket =
- indexBucketLocator.getBucket( applicationId, IndexType.COLLECTION, id, prop );
+ for ( String prop : indexed ) {
- Object rowKey = key( applicationId, collection.getName(), prop, bucket );
+ String bucket = indexBucketLocator
+ .getBucket( id );
- List<HColumn<ByteBuffer, ByteBuffer>> indexCols =
- scanIndexForAllTypes( ko, indexBucketLocator, applicationId, rowKey, id, prop );
+ Object rowKey = key( applicationId, collection.getName(), prop, bucket );
- // loop through the indexed values and verify them as present in
- // our entity_index_entries. If they aren't, we need to delete the
- // from the secondary index, and mark
- // this object for re-index via n update
- for ( HColumn<ByteBuffer, ByteBuffer> index : indexCols ) {
+ List<HColumn<ByteBuffer, ByteBuffer>> indexCols =
+ scanIndexForAllTypes( ko, indexBucketLocator, applicationId, rowKey, id,
+ prop );
- DynamicComposite secondaryIndexValue =
- DynamicComposite.fromByteBuffer( index.getName().duplicate() );
+ // loop through the indexed values and verify them as present in
+ // our entity_index_entries. If they aren't, we need to delete the
+ // from the secondary index, and mark
+ // this object for re-index via n update
+ for ( HColumn<ByteBuffer, ByteBuffer> index : indexCols ) {
- Object code = secondaryIndexValue.get( 0 );
- Object propValue = secondaryIndexValue.get( 1 );
- UUID timestampId = ( UUID ) secondaryIndexValue.get( 3 );
+ DynamicComposite secondaryIndexValue = DynamicComposite.fromByteBuffer( index
+ .getName().duplicate() );
- DynamicComposite existingEntryStart =
- new DynamicComposite( prop, code, propValue, timestampId );
- DynamicComposite existingEntryFinish =
- new DynamicComposite( prop, code, propValue, timestampId );
+ Object code = secondaryIndexValue.get( 0 );
+ Object propValue = secondaryIndexValue.get( 1 );
+ UUID timestampId = ( UUID ) secondaryIndexValue.get( 3 );
- setEqualityFlag( existingEntryFinish, ComponentEquality.GREATER_THAN_EQUAL );
+ DynamicComposite existingEntryStart = new DynamicComposite( prop, code,
+ propValue, timestampId );
+ DynamicComposite existingEntryFinish = new DynamicComposite( prop, code,
+ propValue, timestampId );
- // now search our EntityIndexEntry for previous values, see if
- // they don't match this one
+ setEqualityFlag( existingEntryFinish, ComponentEquality.GREATER_THAN_EQUAL );
- List<HColumn<ByteBuffer, ByteBuffer>> entries =
- cass.getColumns( ko, ENTITY_INDEX_ENTRIES, id, existingEntryStart,
- existingEntryFinish, INDEX_ENTRY_LIST_COUNT, false );
+ // now search our EntityIndexEntry for previous values, see if
+ // they don't match this one
- // we wouldn't find this column in our entity_index_entries
- // audit. Delete it, then mark this entity for update
- if ( entries.size() == 0 ) {
- logger.info(
- "Could not find reference to value '{}' for property '{}' on entity " +
- "{} in collection {}. " + " Forcing reindex",
- new Object[] { propValue, prop, id, collectionName } );
+ List<HColumn<ByteBuffer, ByteBuffer>> entries =
+ cass.getColumns( ko, ENTITY_INDEX_ENTRIES, id, existingEntryStart,
+ existingEntryFinish, INDEX_ENTRY_LIST_COUNT, false );
- addDeleteToMutator( m, ENTITY_INDEX, rowKey, index.getName().duplicate(),
- timestamp );
+ // we wouldn't find this column in our entity_index_entries
+ // audit. Delete it, then mark this entity for update
+ if ( entries.size() == 0 ) {
+ logger.info(
+ "Could not find reference to value '{}' for property '{}' on entity "
+ +
+ "{} in collection {}. " + " Forcing reindex", new Object[] { propValue, prop, id, collectionName } );
- reIndex = true;
- }
+ addDeleteToMutator( m, ENTITY_INDEX, rowKey, index.getName().duplicate(),
+ timestamp );
+
+ reIndex = true;
+ }
- if ( entries.size() > 1 ) {
- logger.info(
- "Found more than 1 entity referencing unique index for property '{}' " +
- "with value " + "'{}'", prop, propValue );
- reIndex = true;
+ if ( entries.size() > 1 ) {
+ logger.info(
+ "Found more than 1 entity referencing unique index for property "
+ + "'{}' "
+ +
+ "with value " + "'{}'", prop, propValue );
+ reIndex = true;
+ }
}
}
- }
- //force this entity to be updated
- if ( reIndex ) {
- Entity entity = em.get( id );
-
- //entity may not exist, but we should have deleted rows from the index
- if ( entity == null ) {
- logger.warn( "Entity with id {} did not exist in app {}", id, applicationId );
- //now execute the cleanup. In this case the entity is gone,
- // so we'll want to remove references from
- // the secondary index
- m.execute();
- continue;
- }
+ //force this entity to be updated
+ if ( reIndex ) {
+ Entity entity = em.get( id );
+
+ //entity may not exist, but we should have deleted rows from the index
+ if ( entity == null ) {
+ logger.warn( "Entity with id {} did not exist in app {}", id, applicationId );
+ //now execute the cleanup. In this case the entity is gone,
+ // so we'll want to remove references from
+ // the secondary index
+ m.execute();
+ continue;
+ }
- logger.info( "Reindex complete for entity with id '{} ", id );
- em.update( entity );
+ logger.info( "Reindex complete for entity with id '{} ", id );
+ em.update( entity );
- //now execute the cleanup. This way if the above update fails,
- // we still have enough data to run again
- // later
- m.execute();
+ //now execute the cleanup. This way if the above update fails,
+ // we still have enough data to run again
+ // later
+ m.execute();
+ }
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to process entity with id '{}'", id, e );
}
- }
- catch ( Exception e ) {
- logger.error( "Unable to process entity with id '{}'", id, e );
}
}
}