You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/03/07 23:21:07 UTC
[2/4] git commit: 61 tests passing, 3 tests skipped.
61 tests passing, 3 tests skipped.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a59ced69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a59ced69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a59ced69
Branch: refs/heads/two-dot-o
Commit: a59ced690e3b6de380797e01bb82087847f6e223
Parents: 24a8f4d
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Mar 7 12:36:25 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Mar 7 12:36:25 2014 -0500
----------------------------------------------------------------------
.../collection/rx/CassandraThreadScheduler.java | 2 +-
stack/corepersistence/queryindex/README.md | 2 +
.../index/impl/EsEntityCollectionIndex.java | 56 ++++++++++++++++----
.../persistence/index/impl/CollectionIT.java | 2 -
.../index/impl/EntityCollectionIndexTest.java | 13 +++--
.../usergrid/persistence/index/impl/GeoIT.java | 22 ++++----
.../persistence/index/impl/IndexIT.java | 8 +--
7 files changed, 69 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a59ced69/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/rx/CassandraThreadScheduler.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/rx/CassandraThreadScheduler.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/rx/CassandraThreadScheduler.java
index 90f3b2d..77cafcc 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/rx/CassandraThreadScheduler.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/rx/CassandraThreadScheduler.java
@@ -65,7 +65,7 @@ public class CassandraThreadScheduler implements Provider<Scheduler> {
final String threadName = "RxCassandraIOThreadPool-" + counter.incrementAndGet();
- LOG.info( "Allocating new IO thread with name {}", threadName );
+ LOG.debug( "Allocating new IO thread with name {}", threadName );
Thread t = new Thread( r, threadName );
t.setDaemon( true );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a59ced69/stack/corepersistence/queryindex/README.md
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/README.md b/stack/corepersistence/queryindex/README.md
index 14426ce..018eef8 100644
--- a/stack/corepersistence/queryindex/README.md
+++ b/stack/corepersistence/queryindex/README.md
@@ -48,6 +48,8 @@ Issues and work remaining
* Better to have index for all, or one per organization?
+* For each index, how many shards? The default five is good enough?
+ * The number of shards = the maximum number of nodes possible
__Work remaining:__
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a59ced69/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityCollectionIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityCollectionIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityCollectionIndex.java
index 74fbf20..17eb1d1 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityCollectionIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityCollectionIndex.java
@@ -17,6 +17,7 @@
*/
package org.apache.usergrid.persistence.index.impl;
+import com.google.common.util.concurrent.AtomicDouble;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import java.io.IOException;
@@ -27,6 +28,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.commons.lang3.time.StopWatch;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
@@ -47,6 +50,7 @@ import org.apache.usergrid.persistence.query.Query;
import org.apache.usergrid.persistence.query.Results;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
@@ -83,14 +87,17 @@ public class EsEntityCollectionIndex implements EntityCollectionIndex {
private final CollectionScope scope;
private final EntityCollectionManager manager;
+ private final AtomicLong indexedCount = new AtomicLong(0L);
+ private final AtomicDouble averageIndexTime = new AtomicDouble(0);
+
public static final String ANALYZED_SUFFIX = "_ug_analyzed";
public static final String GEO_SUFFIX = "_ug_geo";
- public static final String ID_SEPARATOR = "|";
- public static final String ID_SEPARATOR_SPLITTER = "\\|";
+ public static final String DOC_ID_SEPARATOR = "|";
+ public static final String DOC_ID_SEPARATOR_SPLITTER = "\\|";
// These are not allowed in document type names: _ . , | #
- public static final String TYPE_SEPARATOR = "^";
+ public static final String DOC_TYPE_SEPARATOR = "^";
@Inject
public EsEntityCollectionIndex(@Assisted final CollectionScope scope,
@@ -124,12 +131,23 @@ public class EsEntityCollectionIndex implements EntityCollectionIndex {
try {
XContentBuilder mxcb = EsEntityCollectionIndex
- .createDoubleStringIndexMapping(jsonBuilder(), scope.getName());
+ .createDoubleStringIndexMapping(jsonBuilder(), typeName);
PutMappingResponse pmr = admin.indices().preparePutMapping( indexName )
- .setType(scope.getName()).setSource(mxcb).execute().actionGet();
+ .setType( typeName ).setSource(mxcb).execute().actionGet();
+
+ if (!admin.indices().typesExists(new TypesExistsRequest(
+ new String[] { indexName }, typeName )).actionGet().isExists()) {
+ throw new RuntimeException("Type does not exist in index: " + typeName);
+ }
- log.debug("Created new type mapping for scope named: " + scope.getName());
+ GetMappingsResponse gmr = admin.indices().prepareGetMappings( indexName )
+ .addTypes( typeName ).execute().actionGet();
+ if ( gmr.getMappings().isEmpty() ) {
+ throw new RuntimeException("Zero mappings exist for type: " + typeName);
+ }
+
+ log.debug("Created new type mapping for scope: " + scope.getName());
log.debug(" Scope organization: " + scope.getOrganization());
log.debug(" Scope owner: " + scope.getOwner());
log.debug(" Type name: " + typeName );
@@ -148,8 +166,8 @@ public class EsEntityCollectionIndex implements EntityCollectionIndex {
private String createIndexId(Id entityId, UUID version) {
- String sep = ID_SEPARATOR;
StringBuilder sb = new StringBuilder();
+ String sep = DOC_ID_SEPARATOR;
sb.append( entityId.getUuid() ).append(sep);
sb.append( entityId.getType() ).append(sep);
sb.append( version.toString() );
@@ -158,8 +176,8 @@ public class EsEntityCollectionIndex implements EntityCollectionIndex {
public static String createTypeName( CollectionScope scope ) {
- String sep = TYPE_SEPARATOR;
StringBuilder sb = new StringBuilder();
+ String sep = DOC_TYPE_SEPARATOR;
sb.append( scope.getName() ).append(sep);
sb.append( scope.getOwner().getUuid() ).append(sep);
sb.append( scope.getOwner().getType() ).append(sep);
@@ -171,6 +189,12 @@ public class EsEntityCollectionIndex implements EntityCollectionIndex {
public void index(Entity entity) {
+ StopWatch timer = null;
+ if ( log.isDebugEnabled() ) {
+ timer = new StopWatch();
+ timer.start();
+ }
+
if (entity.getId() == null) {
throw new IllegalArgumentException("Cannot index entity with id null");
}
@@ -193,7 +217,19 @@ public class EsEntityCollectionIndex implements EntityCollectionIndex {
irb.execute().actionGet();
- log.debug("Indexed Entity with index id " + indexId);
+ //log.debug("Indexed Entity with index id " + indexId);
+
+ if ( log.isDebugEnabled() ) {
+ timer.stop();
+ double average = averageIndexTime.get();
+ if ( !averageIndexTime.compareAndSet( 0, timer.getTime() ) ) {
+ averageIndexTime.compareAndSet( average, (average + timer.getTime()) / 2.0 );
+ }
+ long count = indexedCount.addAndGet(1);
+ if ( count % 1000 == 0 ) {
+ log.debug("Indexed {} entities, average time {}ms", count, averageIndexTime.get() );
+ }
+ }
}
public void deindex(Entity entity) {
@@ -263,7 +299,7 @@ public class EsEntityCollectionIndex implements EntityCollectionIndex {
for (SearchHit hit : hits.getHits()) {
- String[] idparts = hit.getId().split( ID_SEPARATOR_SPLITTER );
+ String[] idparts = hit.getId().split( DOC_ID_SEPARATOR_SPLITTER );
String id = idparts[0];
String type = idparts[1];
String version = idparts[2];
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a59ced69/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CollectionIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CollectionIT.java
index 3510568..3a8fc2d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CollectionIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CollectionIT.java
@@ -54,11 +54,9 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import org.junit.Before;
import org.junit.ClassRule;
-import org.junit.Ignore;
import org.junit.runner.RunWith;
-@Ignore
@RunWith(JukitoRunner.class)
@UseModules({ TestIndexModule.class })
public class CollectionIT {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a59ced69/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityCollectionIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityCollectionIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityCollectionIndexTest.java
index fedecb3..b4e247e 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityCollectionIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityCollectionIndexTest.java
@@ -49,7 +49,6 @@ import org.jukito.UseModules;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import org.junit.ClassRule;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -71,10 +70,10 @@ public class EntityCollectionIndexTest {
public MigrationManagerRule migrationManagerRule;
@Inject
- public EntityCollectionIndexFactory collectionIndexFactory;
+ public EntityCollectionIndexFactory cif;
@Inject
- public EntityCollectionManagerFactory collectionManagerFactory;
+ public EntityCollectionManagerFactory cmf;
@Test
public void testIndex() throws IOException {
@@ -83,9 +82,9 @@ public class EntityCollectionIndexTest {
Id orgId = new SimpleId("organization");
CollectionScope scope = new CollectionScopeImpl( appId, orgId, "contacts" );
- EntityCollectionManager entityManager = collectionManagerFactory.createCollectionManager( scope );
+ EntityCollectionManager entityManager = cmf.createCollectionManager( scope );
- EntityCollectionIndex entityIndex = collectionIndexFactory.createCollectionIndex( scope );
+ EntityCollectionIndex entityIndex = cif.createCollectionIndex( scope );
InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" );
ObjectMapper mapper = new ObjectMapper();
@@ -123,8 +122,8 @@ public class EntityCollectionIndexTest {
Id orgId = new SimpleId("AutoWorldMagazine");
CollectionScope scope = new CollectionScopeImpl( appId, orgId, "fastcars" );
- EntityCollectionIndex entityIndex = collectionIndexFactory.createCollectionIndex( scope );
- EntityCollectionManager entityManager = collectionManagerFactory.createCollectionManager( scope );
+ EntityCollectionIndex entityIndex = cif.createCollectionIndex( scope );
+ EntityCollectionManager entityManager = cmf.createCollectionManager( scope );
Map entityMap = new HashMap() {{
put("name", "Ferrari 212 Inter");
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a59ced69/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoIT.java
index 51ba458..ee2ceca 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoIT.java
@@ -55,7 +55,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-@Ignore
@RunWith(JukitoRunner.class)
@UseModules({ TestIndexModule.class })
public class GeoIT {
@@ -236,14 +235,14 @@ public class GeoIT {
public void testPointPaging() throws Exception {
Id appId = new SimpleId("testGeo");
- Id orgId = new SimpleId("testOrganization");
+ Id orgId = new SimpleId("testPointPaging");
EntityManagerFacade em = new EntityManagerFacade( orgId, appId,
collectionManagerFactory, collectionIndexFactory );
assertNotNull( em );
// save objects in a diagonal line from -90 -180 to 90 180
- int numEntities = 10;
+ int numEntities = 500;
float minLattitude = -90;
float maxLattitude = 90;
@@ -293,11 +292,12 @@ public class GeoIT {
}
+ @Ignore
@Test
public void testSamePointPaging() throws Exception {
Id appId = new SimpleId("testGeo");
- Id orgId = new SimpleId("testOrganization");
+ Id orgId = new SimpleId("testSamePointPaging");
EntityManagerFacade em = new EntityManagerFacade( orgId, appId,
collectionManagerFactory, collectionIndexFactory );
assertNotNull( em );
@@ -340,11 +340,12 @@ public class GeoIT {
}
+ @Ignore
@Test
public void testDistanceByLimit() throws Exception {
Id appId = new SimpleId("testGeo");
- Id orgId = new SimpleId("testOrganization");
+ Id orgId = new SimpleId("testDistanceByLimit");
EntityManagerFacade em = new EntityManagerFacade( orgId, appId,
collectionManagerFactory, collectionIndexFactory );
assertNotNull( em );
@@ -397,11 +398,12 @@ public class GeoIT {
}
+ @Ignore
@Test
public void testGeoWithIntersection() throws Exception {
Id appId = new SimpleId("testGeo");
- Id orgId = new SimpleId("testOrganization");
+ Id orgId = new SimpleId("testGeoWithIntersection");
EntityManagerFacade em = new EntityManagerFacade( orgId, appId,
collectionManagerFactory, collectionIndexFactory );
assertNotNull( em );
@@ -534,7 +536,7 @@ public class GeoIT {
// }
- public Map<String, Object> getLocation( double latitude, double longitude ) throws Exception {
+ private Map<String, Object> getLocation( double latitude, double longitude ) throws Exception {
Map<String, Object> latlong = new LinkedHashMap<String, Object>();
latlong.put( "latitude", latitude );
latlong.put( "longitude", longitude );
@@ -542,16 +544,16 @@ public class GeoIT {
}
- public void updatePos( EntityManagerFacade em, EntityRef ref, double lat, double lon) throws Exception {
+ private void updatePos( EntityManagerFacade em, EntityRef ref, double lat, double lon) throws Exception {
em.setProperty( ref, "location", lat, lon );
}
- public void updatePos( EntityManagerFacade em, Entity e, double lat, double lon) throws Exception {
+ private void updatePos( EntityManagerFacade em, Entity e, double lat, double lon) throws Exception {
em.setProperty( new SimpleEntityRef( e.getId(), e.getVersion()), "location", lat, lon );
}
- public void setPos( Map<String, Object> data, double latitude, double longitude ) {
+ private void setPos( Map<String, Object> data, double latitude, double longitude ) {
Map<String, Object> latlong = new LinkedHashMap<String, Object>();
latlong.put( "latitude", latitude );
latlong.put( "longitude", longitude );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a59ced69/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexIT.java
index d31a121..9e924cc 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexIT.java
@@ -40,14 +40,13 @@ import org.jukito.JukitoRunner;
import org.jukito.UseModules;
import static org.junit.Assert.assertEquals;
import org.junit.ClassRule;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-@Ignore
+
@RunWith(JukitoRunner.class)
@UseModules({ TestIndexModule.class })
public class IndexIT {
@@ -82,7 +81,6 @@ public class IndexIT {
"X-ray", "Yankee", "Zulu"
};
- //@Ignore // TODO: diagnose why this sometimes hangs and causes all subsequent tests to fail.
@Test
public void testCollectionOrdering() throws Exception {
LOG.info( "testCollectionOrdering" );
@@ -153,7 +151,6 @@ public class IndexIT {
}
- //@Ignore // TODO: diagnose why this sometimes hangs and causes all subsequent tests to fail.
@Test
public void testCollectionFilters() throws Exception {
LOG.info( "testCollectionFilters" );
@@ -265,7 +262,7 @@ public class IndexIT {
LOG.info( JsonUtils.mapToFormattedJsonString( r.getEntities() ) );
assertEquals( 1, r.size() );
- long created = r.getEntity().getVersion().timestamp();
+ long created = r.getEntity().getId().getUuid().timestamp();
Id entityId = r.getEntity().getId();
query = Query.fromQL( "created = " + created );
@@ -275,7 +272,6 @@ public class IndexIT {
assertEquals( entityId, r.getEntity().getId() );
}
- //@Ignore // TODO: diagnose why this sometimes hangs and causes all subsequent tests to fail.
@Test
public void testSecondarySorts() throws Exception {
LOG.info( "testSecondarySorts" );