You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/03/10 19:37:22 UTC
[4/5] incubator-usergrid git commit: Addes Cache to
MapMangerFactorImpl for efficiency
Addes Cache to MapMangerFactorImpl for efficiency
Adds timeout for string storage in our map
Adds time UUID generated cursors to avoid USERGRID-461. Note that the map CF will need to be tuned appropriately. I.E lower gc_grace times since columns are expire and are never deleted. SSD's with key cache, or row+ key cache for spinning disks at the physical tier.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5d95ebba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5d95ebba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5d95ebba
Branch: refs/heads/two-dot-o
Commit: 5d95ebbadc37609c200ea8def435b945c9641bac
Parents: e27f090
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 9 16:14:21 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 9 16:14:21 2015 -0600
----------------------------------------------------------------------
.../persistence/map/MapManagerFactory.java | 2 +
.../persistence/map/guice/MapModule.java | 6 +-
.../map/impl/MapManagerFactoryImpl.java | 67 ++++++++++
stack/corepersistence/queryindex/pom.xml | 16 ++-
.../persistence/index/guice/IndexModule.java | 4 +
.../index/impl/EsEntityIndexFactoryImpl.java | 12 +-
.../index/impl/EsEntityIndexImpl.java | 57 +++++++--
.../persistence/index/impl/IndexingUtils.java | 2 +-
.../persistence/index/utils/StringUtils.java | 126 ++-----------------
.../persistence/index/utils/UUIDUtils.java | 4 -
.../persistence/index/impl/EntityIndexTest.java | 99 ++++++++++++++-
11 files changed, 247 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManagerFactory.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManagerFactory.java
index a60cdfc..81531c9 100644
--- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManagerFactory.java
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManagerFactory.java
@@ -27,4 +27,6 @@ public interface MapManagerFactory {
* Get the map manager
*/
public MapManager createMapManager( final MapScope scope );
+
+ void invalidate();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/guice/MapModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/guice/MapModule.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/guice/MapModule.java
index d167151..5c46c87 100644
--- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/guice/MapModule.java
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/guice/MapModule.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.persistence.map.guice;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.impl.MapManagerFactoryImpl;
import org.apache.usergrid.persistence.map.impl.MapManagerImpl;
import org.apache.usergrid.persistence.map.impl.MapSerialization;
import org.apache.usergrid.persistence.map.impl.MapSerializationImpl;
@@ -43,12 +44,11 @@ public class MapModule extends AbstractModule {
protected void configure() {
// create a guice factory for getting our collection manager
- install( new FactoryModuleBuilder().implement( MapManager.class, MapManagerImpl.class )
- .build( MapManagerFactory.class ) );
-
+ bind(MapManagerFactory.class).to( MapManagerFactoryImpl.class );
bind( MapSerialization.class).to( MapSerializationImpl.class );
+
Multibinder<Migration> migrationBinding = Multibinder.newSetBinder( binder(), Migration.class );
migrationBinding.addBinding().to( Key.get( MapSerialization.class ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerFactoryImpl.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerFactoryImpl.java
new file mode 100644
index 0000000..e69a02e
--- /dev/null
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapManagerFactoryImpl.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.map.impl;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.netflix.astyanax.Execution;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.MapScope;
+
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Returns map managers, built to handle caching
+ */
+@Singleton
+public class MapManagerFactoryImpl implements MapManagerFactory {
+ private final MapSerialization mapSerialization;
+ private LoadingCache<MapScope, MapManager> mmCache =
+ CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<MapScope, MapManager>() {
+ public MapManager load( MapScope scope ) {
+ return new MapManagerImpl(scope,mapSerialization);
+ }
+ } );
+
+ @Inject
+ public MapManagerFactoryImpl(final MapSerialization mapSerialization){
+
+ this.mapSerialization = mapSerialization;
+ }
+
+ @Override
+ public MapManager createMapManager(MapScope scope) {
+ Preconditions.checkNotNull(scope);
+ try{
+ return mmCache.get(scope);
+ }catch (ExecutionException ee){
+ throw new RuntimeException(ee);
+ }
+ }
+
+ @Override
+ public void invalidate() {
+ mmCache.invalidateAll();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/pom.xml b/stack/corepersistence/queryindex/pom.xml
index 8604316..f6ae718 100644
--- a/stack/corepersistence/queryindex/pom.xml
+++ b/stack/corepersistence/queryindex/pom.xml
@@ -50,10 +50,10 @@
<artifactId>chop-maven-plugin</artifactId>
<version>${chop.version}</version>
-
+
NOTE: you should be putting most of these variables into your settings.xml
as an automatically activated profile.
-
+
<configuration>
<accessKey>${aws.s3.key}</accessKey>
@@ -74,11 +74,11 @@
<runnerKeyPairName>${runner.keypair.name}</runnerKeyPairName>
<runnerCount>6</runnerCount>
<securityGroupExceptions>
-
+
Add your own IP address as an exception to allow access
but please do this in the settings.xml file .. essentially
all parameters should be in the settings.xml file.
-
+
<param>${myip.address}/32:24981</param>
<param>${myip.address}/32:22</param>
</securityGroupExceptions>
@@ -99,6 +99,14 @@
<type>jar</type>
</dependency>
+
+ <dependency>
+ <groupId>${project.parent.groupId}</groupId>
+ <artifactId>map</artifactId>
+ <version>${project.version}</version>
+ <type>jar</type>
+ </dependency>
+
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index 6fee17e..ebd9098 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -27,6 +27,8 @@ import org.apache.usergrid.persistence.index.impl.EsEntityIndexFactoryImpl;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferConsumerImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferProducerImpl;
+import org.apache.usergrid.persistence.map.guice.MapModule;
+
import org.safehaus.guicyfig.GuicyFigModule;
@@ -38,6 +40,8 @@ public class IndexModule extends AbstractModule {
// install our configuration
install(new GuicyFigModule(IndexFig.class));
+ install(new MapModule());
+
bind(EntityIndexFactory.class).to( EsEntityIndexFactoryImpl.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
index 3a9f790..8af309d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
@@ -23,7 +23,6 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -31,6 +30,7 @@ import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexBufferProducer;
import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
import java.util.concurrent.ExecutionException;
@@ -44,23 +44,29 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
private final EsIndexCache indexCache;
private final IndexBufferProducer indexBatchBufferProducer;
private final MetricsFactory metricsFactory;
+ private final MapManagerFactory mapManagerFactory;
+ private final IndexFig indexFig;
private LoadingCache<ApplicationScope, EntityIndex> eiCache =
CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<ApplicationScope, EntityIndex>() {
public EntityIndex load( ApplicationScope scope ) {
- return new EsEntityIndexImpl(scope,config, indexBatchBufferProducer, provider,indexCache, metricsFactory);
+ return new EsEntityIndexImpl(scope,config, indexBatchBufferProducer, provider,indexCache, metricsFactory,
+ mapManagerFactory, indexFig );
}
} );
@Inject
public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider, final EsIndexCache indexCache,
final IndexBufferProducer indexBatchBufferProducer,
- final MetricsFactory metricsFactory ){
+ final MetricsFactory metricsFactory, final MapManagerFactory mapManagerFactory,
+ final IndexFig indexFig ){
this.config = config;
this.provider = provider;
this.indexCache = indexCache;
this.indexBatchBufferProducer = indexBatchBufferProducer;
this.metricsFactory = metricsFactory;
+ this.mapManagerFactory = mapManagerFactory;
+ this.indexFig = indexFig;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index c9f5590..c92b299 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.persistence.index.impl;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
@@ -38,6 +39,11 @@ import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.MapScope;
+import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
@@ -81,6 +87,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
@@ -99,6 +106,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private final IndexIdentifier.IndexAlias alias;
private final IndexIdentifier indexIdentifier;
private final IndexBufferProducer indexBatchBufferProducer;
+ private final IndexFig indexFig;
private final Timer addTimer;
private final Timer addWriteAliasTimer;
private final Timer addReadAliasTimer;
@@ -141,14 +149,18 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private Timer allVersionsTimer;
private Timer deletePreviousTimer;
+ private final MapManager mapManager;
+
// private final Timer indexTimer;
@Inject
public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config,
final IndexBufferProducer indexBatchBufferProducer, final EsProvider provider,
- final EsIndexCache indexCache, final MetricsFactory metricsFactory) {
+ final EsIndexCache indexCache, final MetricsFactory metricsFactory,
+ final MapManagerFactory mapManagerFactory, final IndexFig indexFig ) {
this.indexBatchBufferProducer = indexBatchBufferProducer;
+ this.indexFig = indexFig;
ValidationUtils.validateApplicationScope( appScope );
this.applicationScope = appScope;
this.esProvider = provider;
@@ -181,6 +193,10 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.all.versions.timer" );
this.deletePreviousTimer = metricsFactory
.getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.previous.versions.timer" );
+
+ final MapScope mapScope = new MapScopeImpl( appScope.getApplication(), "cursorcache" );
+
+ mapManager = mapManagerFactory.createMapManager( mapScope );
}
@Override
@@ -467,17 +483,26 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
failureMonitor.success();
}
else {
- String scrollId = query.getCursor();
- if ( scrollId.startsWith( "\"" ) ) {
- scrollId = scrollId.substring( 1 );
+ String userCursorString = query.getCursor();
+ if ( userCursorString.startsWith( "\"" ) ) {
+ userCursorString = userCursorString.substring( 1 );
}
- if ( scrollId.endsWith( "\"" ) ) {
- scrollId = scrollId.substring( 0, scrollId.length() - 1 );
+ if ( userCursorString.endsWith( "\"" ) ) {
+ userCursorString = userCursorString.substring( 0, userCursorString.length() - 1 );
}
- logger.debug( "Executing query with cursor: {} ", scrollId );
+
+ //now get the cursor from the map and validate
+ final String esScrollCursor = mapManager.getString( userCursorString );
+
+ Preconditions.checkArgument(esScrollCursor != null, "Could not find a cursor for the value '{}' ", esScrollCursor);
+
+
+
+ logger.debug( "Executing query with cursor: {} ", esScrollCursor );
+
SearchScrollRequestBuilder ssrb = esProvider.getClient()
- .prepareSearchScroll(scrollId).setScroll( cursorTimeout + "m" );
+ .prepareSearchScroll(esScrollCursor).setScroll( cursorTimeout + "m" );
try {
//Added For Graphite Metrics
@@ -524,8 +549,20 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
CandidateResults candidateResults = new CandidateResults( query, candidates );
if ( candidates.size() >= query.getLimit() ) {
- candidateResults.setCursor( searchResponse.getScrollId() );
- logger.debug(" Cursor = " + searchResponse.getScrollId());
+ //USERGRID-461 our cursor is getting too large, map it to a new time UUID
+
+ final String userCursorString = org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
+
+ final String esScrollCursor = searchResponse.getScrollId();
+
+ //now set this into our map module
+ final int minutes = indexFig.getQueryCursorTimeout();
+
+ //just truncate it, we'll never hit a long value anyway
+ mapManager.putString( userCursorString, esScrollCursor, ( int ) TimeUnit.MINUTES.toSeconds( minutes ) );
+
+ candidateResults.setCursor( userCursorString );
+ logger.debug(" User cursor = {}, Cursor = {} ", userCursorString, esScrollCursor);
}
return candidateResults;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index be0c96c..ffd98e9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -152,7 +152,7 @@ public class IndexingUtils {
.startObject()
/** add routing "_routing":{ "required":false, "path":"ug_entityId" **/
- //.startObject("_routing").field("required",true).field("path",ENTITYID_ID_FIELDNAME).endObject()
+ .startObject("_routing").field("required",true).field("path",ENTITYID_ID_FIELDNAME).endObject()
.startArray("dynamic_templates")
// we need most specific mappings first since it's a stop on match algorithm
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
index 8aabbbf..a567594 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/StringUtils.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.persistence.index.utils;
import java.util.Arrays;
+import java.util.UUID;
import org.apache.commons.io.IOUtils;
@@ -32,37 +33,6 @@ public class StringUtils extends org.apache.commons.lang.StringUtils {
private static final Logger LOG = LoggerFactory.getLogger( StringUtils.class );
- public static Object lower( Object obj ) {
- if ( !( obj instanceof String ) ) {
- return obj;
- }
- return ( ( String ) obj ).toLowerCase();
- }
-
-
- public static String stringOrSubstringAfterLast( String str, char c ) {
- if ( str == null ) {
- return null;
- }
- int i = str.lastIndexOf( c );
- if ( i != -1 ) {
- return str.substring( i + 1 );
- }
- return str;
- }
-
-
- public static String stringOrSubstringBeforeLast( String str, char c ) {
- if ( str == null ) {
- return null;
- }
- int i = str.lastIndexOf( c );
- if ( i != -1 ) {
- return str.substring( 0, i );
- }
- return str;
- }
-
public static String stringOrSubstringBeforeFirst( String str, char c ) {
if ( str == null ) {
@@ -76,97 +46,17 @@ public class StringUtils extends org.apache.commons.lang.StringUtils {
}
- public static String stringOrSubstringAfterFirst( String str, char c ) {
- if ( str == null ) {
- return null;
- }
- int i = str.indexOf( c );
- if ( i != -1 ) {
- return str.substring( i + 1 );
- }
- return str;
- }
-
-
- public static String compactWhitespace( String str ) {
- if ( str == null ) {
- return null;
- }
- boolean prevWS = false;
- StringBuilder builder = new StringBuilder();
- for ( int i = 0; i < str.length(); i++ ) {
- char c = str.charAt( i );
- if ( Character.isWhitespace( c ) ) {
- if ( !prevWS ) {
- builder.append( ' ' );
- }
- prevWS = true;
- }
- else {
- prevWS = false;
- builder.append( c );
- }
- }
- return builder.toString().trim();
- }
-
-
- /** @return new string with replace applied */
- public static String replaceAll( String source, String find, String replace ) {
- if ( source == null ) {
- return null;
- }
- while ( true ) {
- String old = source;
- source = source.replaceAll( find, replace );
- if ( source.equals( old ) ) {
- return source;
- }
- }
- }
-
-
public static String toString( Object obj ) {
return string( obj );
}
- public static String toStringFormat( Object obj, String format ) {
- if ( obj != null ) {
- if ( format != null ) {
- if ( obj.getClass().isArray() ) {
- return String.format( format, Arrays.toString( ( Object[] ) obj ) );
- }
- return String.format( format, string( obj ) );
- }
- else {
- return string( obj );
- }
- }
- return "";
- }
-
-
- public static boolean isString( Object obj ) {
- return obj instanceof String;
- }
-
-
- public static boolean isStringOrNull( Object obj ) {
- if ( obj == null ) {
- return true;
- }
- return obj instanceof String;
- }
-
-
- public static String readClasspathFileAsString( String filePath ) {
- try {
- return IOUtils.toString( StringUtils.class.getResourceAsStream( filePath ) );
- }
- catch ( Exception e ) {
- LOG.error( "Error getting file from classpath: " + filePath, e );
- }
- return null;
+ /**
+ * Remove dashes from our uuid
+ * @param uuid
+ * @return
+ */
+ public static String sanitizeUUID(final UUID uuid){
+ return uuid.toString().replace( "-", "" );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
index fdffaef..b9b407b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/utils/UUIDUtils.java
@@ -57,10 +57,6 @@ public class UUIDUtils {
private static ReentrantLock tsLock = new ReentrantLock( true );
- public static final UUID MIN_TIME_UUID = UUID.fromString( "00000000-0000-1000-8000-000000000000" );
-
- public static final UUID MAX_TIME_UUID = UUID.fromString( "ffffffff-ffff-1fff-bfff-ffffffffffff" );
-
public static final UUID ZERO_UUID = new UUID( 0, 0 );
private static long timestampMillisNow = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d95ebba/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 70ae8c5..a3332a8 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -24,10 +24,7 @@ import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.index.*;
-import org.apache.usergrid.persistence.index.query.CandidateResult;
-import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
import org.apache.usergrid.persistence.model.field.UUIDField;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -289,8 +286,8 @@ public class EntityIndexTest extends BaseIT {
entityIndex.createBatch().index(indexScope , entity ).execute().get();
entityIndex.refresh();
- CandidateResults candidateResults = entityIndex.search( indexScope, SearchTypes.fromTypes(entity.getId().getType()),
- Query.fromQL( "name contains 'Ferrari*'" ) );
+ CandidateResults candidateResults = entityIndex.search( indexScope,
+ SearchTypes.fromTypes( entity.getId().getType() ), Query.fromQL( "name contains 'Ferrari*'" ) );
assertEquals( 1, candidateResults.size() );
EntityIndexBatch batch = entityIndex.createBatch();
@@ -593,6 +590,98 @@ public class EntityIndexTest extends BaseIT {
assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() );
assertNotEquals( "cluster should be ready now", Health.RED, ei.getClusterHealth() );
}
+
+
+ @Test
+ public void testCursorFormat() throws Exception {
+
+ Id appId = new SimpleId( "application" );
+ Id ownerId = new SimpleId( "owner" );
+
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+ IndexScope indexScope = new IndexScopeImpl( ownerId, "users" );
+
+
+ EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+ entityIndex.initializeIndex();
+
+ final EntityIndexBatch batch = entityIndex.createBatch();
+
+
+ final int size = 10;
+
+ final List<Id> entities = new ArrayList<>( size );
+
+
+ for ( int i = 0; i < size; i++ ) {
+ final String middleName = "middleName" + UUIDUtils.newTimeUUID();
+ Map<String, Object> properties = new LinkedHashMap<String, Object>();
+ properties.put( "username", "edanuff" );
+ properties.put( "email", "ed@anuff.com" );
+ properties.put( "middlename", middleName );
+
+ Map entityMap = new HashMap() {{
+ put( "username", "edanuff" );
+ put( "email", "ed@anuff.com" );
+ put( "middlename", middleName );
+ }};
+
+ final Id userId = new SimpleId( "user" );
+
+ Entity user = EntityIndexMapUtils.fromMap( entityMap );
+ EntityUtils.setId( user, userId );
+ EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
+
+ user.setField( new UUIDField( IndexingUtils.ENTITYID_ID_FIELDNAME, UUIDGenerator.newTimeUUID() ) );
+
+ entities.add( userId );
+
+
+ batch.index( indexScope, user );
+ }
+
+
+ batch.execute().get();
+ entityIndex.refresh();
+
+
+ final int limit = 1;
+
+
+ final int expectedPages = size / limit;
+
+
+ String cursor = null;
+
+ for ( int i = 0; i < expectedPages; i++ ) {
+ //**
+ final Query query = Query.fromQL( "select *" );
+ query.setLimit( limit );
+
+ if ( cursor != null ) {
+ query.setCursor( cursor );
+ }
+
+ final CandidateResults results = entityIndex.search( indexScope, SearchTypes.allTypes(), query );
+
+ assertTrue( results.hasCursor() );
+
+ cursor = results.getCursor();
+
+ assertEquals("Should be 16 bytes as hex", 32, cursor.length());
+
+
+
+
+ assertEquals( 1, results.size() );
+
+
+ assertEquals( results.get( 0 ).getId(), entities.get( i ) );
+ }
+ }
+
+
}