You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/11/06 01:14:53 UTC
[07/50] git commit: Updated client to use transport client and start
seperate ES instance during tests, not use embedded node
Updated client to use transport client and start seperate ES instance during tests, not use embedded node
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/25f7c6f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/25f7c6f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/25f7c6f4
Branch: refs/heads/two-dot-o
Commit: 25f7c6f4b59c59062e6df0def2ba5ebe4c18f917
Parents: d9374be
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 31 14:18:45 2014 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 31 14:18:45 2014 -0600
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 5 +--
.../usergrid/persistence/index/IndexFig.java | 4 +--
.../index/impl/EsEntityIndexImpl.java | 38 ++++++++++++++------
.../index/impl/ElasticSearchResource.java | 13 ++++++-
.../rest/filters/HealthCheckFilter.java | 1 -
stack/rest/src/main/webapp/WEB-INF/web.xml | 2 +-
.../java/org/apache/usergrid/rest/ITSetup.java | 8 ++---
.../org/apache/usergrid/rest/RestITSuite.java | 4 +--
.../org/apache/usergrid/ServiceITSetupImpl.java | 1 +
.../org/apache/usergrid/ServiceITSuite.java | 3 +-
10 files changed, 55 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 43e3920..a2d73be 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -59,8 +59,9 @@ elasticsearch.cluster_name=elasticsearch
elasticsearch.index_prefix=usergrid
elasticsearch.hosts=127.0.0.1
elasticsearch.port=9300
-elasticsearch.number_shards=1
-elasticsearch.number_replicas=1
+#We don't want to overwrite, let the defaults be used
+#elasticsearch.number_shards=1
+#elasticsearch.number_replicas=1
index.query.limit.default=1000
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index ebf0c21..0f405f3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -93,13 +93,13 @@ public interface IndexFig extends GuicyFig {
@Key( ELASTICSEARCH_NODENAME )
public String getNodeName();
- @Default("1")
+ @Default("6")
@Key( ELASTICSEARCH_NUMBER_OF_SHARDS )
public int getNumberOfShards();
@Default("1")
@Key( ELASTICSEARCH_NUMBER_OF_REPLICAS )
- public int numberOfReplicas();
+ public int getNumberOfReplicas();
@Default( "20" )
@Key( ELASTICSEARCH_FAIL_REFRESH )
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 135822e..c2334eb 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -24,6 +24,10 @@ import java.util.List;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
@@ -31,6 +35,8 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.FilterBuilder;
@@ -132,8 +138,18 @@ public class EsEntityIndexImpl implements EntityIndex {
createMappings();
}
- AdminClient admin = esProvider.getClient().admin();
- CreateIndexResponse cir = admin.indices().prepareCreate( indexName ).execute().actionGet();
+ final AdminClient admin = esProvider.getClient().admin();
+
+ final int numberOfShards = config.getNumberOfShards();
+ final int numberOfReplicas = config.getNumberOfReplicas();
+
+ Settings settings = ImmutableSettings.settingsBuilder()
+ .put("index.number_of_shards", numberOfShards )
+ .put("index.number_of_replicas", numberOfReplicas).build();
+
+
+ final CreateIndexResponse cir = admin.indices().prepareCreate( indexName ).setSettings( settings ).execute().actionGet();
+
logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
// create the document, this ensures the index is ready
@@ -169,7 +185,7 @@ public class EsEntityIndexImpl implements EntityIndex {
esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD ).get();
- log.info( "Successfully created new document with docId {} in index {} and type {}", tempId, indexName,
+ logger.info( "Successfully created new document with docId {} in index {} and type {}", tempId, indexName,
VERIFY_TYPE );
logger.info( "Successfully created new document with docId {} in index {} and type {}",
tempId, indexName, VERIFY_TYPE );
@@ -178,7 +194,7 @@ public class EsEntityIndexImpl implements EntityIndex {
esProvider.getClient().prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE ).setQuery( MATCH_ALL_QUERY_BUILDER )
.get();
- log.info( "Successfully deleted all documents in index {} and type {}", indexName, VERIFY_TYPE );
+ logger.info( "Successfully deleted all documents in index {} and type {}", indexName, VERIFY_TYPE );
logger.info( "Successfully deleted all documents in index {} and type {}",
indexName, VERIFY_TYPE );
@@ -349,7 +365,7 @@ public class EsEntityIndexImpl implements EntityIndex {
public boolean doOp() {
try {
esProvider.getClient().admin().indices().prepareRefresh( indexName ).execute().actionGet();
- log.debug( "Refreshed index: " + indexName );
+ logger.debug( "Refreshed index: " + indexName );
return true;
}
catch ( IndexMissingException e ) {
@@ -421,17 +437,19 @@ public class EsEntityIndexImpl implements EntityIndex {
try {
ClusterHealthResponse health =
- client.admin().cluster().health( new ClusterHealthRequest() ).get();
-
- if ( health.getStatus().equals( ClusterHealthStatus.GREEN ) ) {
- return true;
+ esProvider.getClient().admin().cluster().health( new ClusterHealthRequest() ).get();
+
+ //we only check red, that indicates something is wrong with one of the index on the cluster
+ //TODO, not sure we even want to do this. If 1 index is broken, that app may be broken, but others will function correctly. The status always is the worst index in the cluster
+ if ( health.getStatus().equals( ClusterHealthStatus.RED ) ) {
+ return false;
}
}
catch (Exception ex) {
logger.error("Error connecting to ElasticSearch", ex);
}
- return false ;
+ return true ;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
index 761f1ff..d9f6ca9 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
@@ -49,6 +49,8 @@ public class ElasticSearchResource extends EnvironResource {
private static Node node;
+ private static int port;
+
public ElasticSearchResource() {
@@ -62,6 +64,13 @@ public class ElasticSearchResource extends EnvironResource {
}
+
+
+ public static int getPort() {
+ return port;
+ }
+
+
public synchronized ElasticSearchResource startEs(){
if ( node != null ) {
return this;
@@ -69,7 +78,7 @@ public class ElasticSearchResource extends EnvironResource {
//override the system properties in the Archiaus env
- int port = AvailablePortFinder.getNextAvailable( 9300 );
+ port = AvailablePortFinder.getNextAvailable( 9300 );
final String host = "127.0.0.1";
System.setProperty( IndexFig.ELASTICSEARCH_HOSTS, host );
@@ -128,6 +137,8 @@ public class ElasticSearchResource extends EnvironResource {
}
+
+
/**
* Uses a project.properties file that Maven does substitution on to to replace the value of a property with the
* path to the Maven build directory (a.k.a. target). It then uses this path to generate a random String which it
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java b/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java
index 4ba9b09..721c321 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java
@@ -57,7 +57,6 @@ public class HealthCheckFilter implements Filter {
throw new RuntimeException("Error connecting to query index");
}
- throw new RuntimeException("Foo!");
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/rest/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/web.xml b/stack/rest/src/main/webapp/WEB-INF/web.xml
index c50251b..dabb88e 100644
--- a/stack/rest/src/main/webapp/WEB-INF/web.xml
+++ b/stack/rest/src/main/webapp/WEB-INF/web.xml
@@ -39,7 +39,7 @@
<filter-mapping>
<filter-name>healthCheckFilter</filter-name>
- <url-pattern>/*</url-pattern>
+ <url-pattern>/status</url-pattern>
</filter-mapping>
<filter>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java b/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java
index 3b2cd87..0d6a957 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java
@@ -17,7 +17,6 @@
package org.apache.usergrid.rest;
-import org.apache.usergrid.ElasticSearchResource;
import java.net.URI;
import java.util.Properties;
@@ -31,6 +30,8 @@ import org.apache.usergrid.management.ApplicationCreator;
import org.apache.usergrid.management.ManagementService;
import org.apache.usergrid.persistence.EntityManagerFactory;
import static org.apache.usergrid.persistence.index.impl.EsProvider.LOCAL_ES_PORT_PROPNAME;
+
+import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.security.providers.SignInProviderFactory;
import org.apache.usergrid.security.tokens.TokenService;
import org.apache.usergrid.services.ServiceManagerFactory;
@@ -77,7 +78,7 @@ public class ITSetup extends ExternalResource {
tomcatResource = TomcatResource.instance;
tomcatResource.setWebAppsPath( "src/main/webapp" );
- elasticSearchResource = ElasticSearchResource.instance;
+ elasticSearchResource = new ElasticSearchResource().startEs();
}
@@ -93,7 +94,6 @@ public class ITSetup extends ExternalResource {
synchronized ( cassandraResource ) {
super.before();
- elasticSearchResource.before();
emf = cassandraResource.getBean( EntityManagerFactory.class );
smf = cassandraResource.getBean( ServiceManagerFactory.class );
@@ -133,7 +133,7 @@ public class ITSetup extends ExternalResource {
protected void after() {
emf.flushEntityManagerCaches();
tomcatResource.after();
- elasticSearchResource.after();
+// elasticSearchResource.after();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java b/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
index 1d7aae0..788a4b3 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
@@ -16,12 +16,12 @@
*/
package org.apache.usergrid.rest;
-import org.apache.usergrid.ElasticSearchResource;
import org.junit.ClassRule;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.apache.usergrid.cassandra.CassandraResource;
import org.apache.usergrid.cassandra.Concurrent;
+import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.rest.applications.events.ApplicationRequestCounterIT;
import org.apache.usergrid.rest.applications.collection.devices.DevicesResourceIT;
import org.apache.usergrid.rest.applications.assets.AssetResourceIT;
@@ -57,7 +57,7 @@ import org.apache.usergrid.rest.management.organizations.AdminEmailEncodingIT;
public class RestITSuite {
@ClassRule
- public static ElasticSearchResource elasticSearchResource = ElasticSearchResource.instance;
+ public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource().startEs();
@ClassRule
public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
index 0b99cab..216e0a6 100644
--- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
@@ -29,6 +29,7 @@ import org.apache.usergrid.management.ApplicationCreator;
import org.apache.usergrid.management.ManagementService;
import org.apache.usergrid.management.export.ExportService;
import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.security.providers.SignInProviderFactory;
import org.apache.usergrid.security.tokens.TokenService;
import org.apache.usergrid.services.ServiceManagerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
index 4f2c151..d4e9751 100644
--- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
+++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
@@ -28,6 +28,7 @@ import org.apache.usergrid.management.RoleIT;
import org.apache.usergrid.management.cassandra.ApplicationCreatorIT;
import org.apache.usergrid.management.cassandra.ExportServiceIT;
import org.apache.usergrid.management.cassandra.ManagementServiceIT;
+import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.security.providers.FacebookProviderIT;
import org.apache.usergrid.security.providers.PingIdentityProviderIT;
import org.apache.usergrid.services.ActivitiesServiceIT;
@@ -59,5 +60,5 @@ public class ServiceITSuite {
public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts();
@ClassRule
- public static ElasticSearchResource elasticSearchResource = ElasticSearchResource.instance;
+ public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource().startEs();
}