You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/10/31 21:19:06 UTC
[1/2] git commit: Updated client to use transport client and start
seperate ES instance during tests, not use embedded node
Repository: incubator-usergrid
Updated Branches:
refs/heads/es-client-changes d9374bed0 -> 5a5377b38
Updated client to use transport client and start seperate ES instance during tests, not use embedded node
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/25f7c6f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/25f7c6f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/25f7c6f4
Branch: refs/heads/es-client-changes
Commit: 25f7c6f4b59c59062e6df0def2ba5ebe4c18f917
Parents: d9374be
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 31 14:18:45 2014 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 31 14:18:45 2014 -0600
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 5 +--
.../usergrid/persistence/index/IndexFig.java | 4 +--
.../index/impl/EsEntityIndexImpl.java | 38 ++++++++++++++------
.../index/impl/ElasticSearchResource.java | 13 ++++++-
.../rest/filters/HealthCheckFilter.java | 1 -
stack/rest/src/main/webapp/WEB-INF/web.xml | 2 +-
.../java/org/apache/usergrid/rest/ITSetup.java | 8 ++---
.../org/apache/usergrid/rest/RestITSuite.java | 4 +--
.../org/apache/usergrid/ServiceITSetupImpl.java | 1 +
.../org/apache/usergrid/ServiceITSuite.java | 3 +-
10 files changed, 55 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 43e3920..a2d73be 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -59,8 +59,9 @@ elasticsearch.cluster_name=elasticsearch
elasticsearch.index_prefix=usergrid
elasticsearch.hosts=127.0.0.1
elasticsearch.port=9300
-elasticsearch.number_shards=1
-elasticsearch.number_replicas=1
+#We don't want to overwrite, let the defaults be used
+#elasticsearch.number_shards=1
+#elasticsearch.number_replicas=1
index.query.limit.default=1000
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index ebf0c21..0f405f3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -93,13 +93,13 @@ public interface IndexFig extends GuicyFig {
@Key( ELASTICSEARCH_NODENAME )
public String getNodeName();
- @Default("1")
+ @Default("6")
@Key( ELASTICSEARCH_NUMBER_OF_SHARDS )
public int getNumberOfShards();
@Default("1")
@Key( ELASTICSEARCH_NUMBER_OF_REPLICAS )
- public int numberOfReplicas();
+ public int getNumberOfReplicas();
@Default( "20" )
@Key( ELASTICSEARCH_FAIL_REFRESH )
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 135822e..c2334eb 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -24,6 +24,10 @@ import java.util.List;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
@@ -31,6 +35,8 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.FilterBuilder;
@@ -132,8 +138,18 @@ public class EsEntityIndexImpl implements EntityIndex {
createMappings();
}
- AdminClient admin = esProvider.getClient().admin();
- CreateIndexResponse cir = admin.indices().prepareCreate( indexName ).execute().actionGet();
+ final AdminClient admin = esProvider.getClient().admin();
+
+ final int numberOfShards = config.getNumberOfShards();
+ final int numberOfReplicas = config.getNumberOfReplicas();
+
+ Settings settings = ImmutableSettings.settingsBuilder()
+ .put("index.number_of_shards", numberOfShards )
+ .put("index.number_of_replicas", numberOfReplicas).build();
+
+
+ final CreateIndexResponse cir = admin.indices().prepareCreate( indexName ).setSettings( settings ).execute().actionGet();
+
logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
// create the document, this ensures the index is ready
@@ -169,7 +185,7 @@ public class EsEntityIndexImpl implements EntityIndex {
esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD ).get();
- log.info( "Successfully created new document with docId {} in index {} and type {}", tempId, indexName,
+ logger.info( "Successfully created new document with docId {} in index {} and type {}", tempId, indexName,
VERIFY_TYPE );
logger.info( "Successfully created new document with docId {} in index {} and type {}",
tempId, indexName, VERIFY_TYPE );
@@ -178,7 +194,7 @@ public class EsEntityIndexImpl implements EntityIndex {
esProvider.getClient().prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE ).setQuery( MATCH_ALL_QUERY_BUILDER )
.get();
- log.info( "Successfully deleted all documents in index {} and type {}", indexName, VERIFY_TYPE );
+ logger.info( "Successfully deleted all documents in index {} and type {}", indexName, VERIFY_TYPE );
logger.info( "Successfully deleted all documents in index {} and type {}",
indexName, VERIFY_TYPE );
@@ -349,7 +365,7 @@ public class EsEntityIndexImpl implements EntityIndex {
public boolean doOp() {
try {
esProvider.getClient().admin().indices().prepareRefresh( indexName ).execute().actionGet();
- log.debug( "Refreshed index: " + indexName );
+ logger.debug( "Refreshed index: " + indexName );
return true;
}
catch ( IndexMissingException e ) {
@@ -421,17 +437,19 @@ public class EsEntityIndexImpl implements EntityIndex {
try {
ClusterHealthResponse health =
- client.admin().cluster().health( new ClusterHealthRequest() ).get();
-
- if ( health.getStatus().equals( ClusterHealthStatus.GREEN ) ) {
- return true;
+ esProvider.getClient().admin().cluster().health( new ClusterHealthRequest() ).get();
+
+ //we only check red, that indicates something is wrong with one of the index on the cluster
+ //TODO, not sure we even want to do this. If 1 index is broken, that app may be broken, but others will function correctly. The status always is the worst index in the cluster
+ if ( health.getStatus().equals( ClusterHealthStatus.RED ) ) {
+ return false;
}
}
catch (Exception ex) {
logger.error("Error connecting to ElasticSearch", ex);
}
- return false ;
+ return true ;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
index 761f1ff..d9f6ca9 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
@@ -49,6 +49,8 @@ public class ElasticSearchResource extends EnvironResource {
private static Node node;
+ private static int port;
+
public ElasticSearchResource() {
@@ -62,6 +64,13 @@ public class ElasticSearchResource extends EnvironResource {
}
+
+
+ public static int getPort() {
+ return port;
+ }
+
+
public synchronized ElasticSearchResource startEs(){
if ( node != null ) {
return this;
@@ -69,7 +78,7 @@ public class ElasticSearchResource extends EnvironResource {
//override the system properties in the Archiaus env
- int port = AvailablePortFinder.getNextAvailable( 9300 );
+ port = AvailablePortFinder.getNextAvailable( 9300 );
final String host = "127.0.0.1";
System.setProperty( IndexFig.ELASTICSEARCH_HOSTS, host );
@@ -128,6 +137,8 @@ public class ElasticSearchResource extends EnvironResource {
}
+
+
/**
* Uses a project.properties file that Maven does substitution on to to replace the value of a property with the
* path to the Maven build directory (a.k.a. target). It then uses this path to generate a random String which it
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java b/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java
index 4ba9b09..721c321 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/filters/HealthCheckFilter.java
@@ -57,7 +57,6 @@ public class HealthCheckFilter implements Filter {
throw new RuntimeException("Error connecting to query index");
}
- throw new RuntimeException("Foo!");
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/rest/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/web.xml b/stack/rest/src/main/webapp/WEB-INF/web.xml
index c50251b..dabb88e 100644
--- a/stack/rest/src/main/webapp/WEB-INF/web.xml
+++ b/stack/rest/src/main/webapp/WEB-INF/web.xml
@@ -39,7 +39,7 @@
<filter-mapping>
<filter-name>healthCheckFilter</filter-name>
- <url-pattern>/*</url-pattern>
+ <url-pattern>/status</url-pattern>
</filter-mapping>
<filter>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java b/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java
index 3b2cd87..0d6a957 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java
@@ -17,7 +17,6 @@
package org.apache.usergrid.rest;
-import org.apache.usergrid.ElasticSearchResource;
import java.net.URI;
import java.util.Properties;
@@ -31,6 +30,8 @@ import org.apache.usergrid.management.ApplicationCreator;
import org.apache.usergrid.management.ManagementService;
import org.apache.usergrid.persistence.EntityManagerFactory;
import static org.apache.usergrid.persistence.index.impl.EsProvider.LOCAL_ES_PORT_PROPNAME;
+
+import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.security.providers.SignInProviderFactory;
import org.apache.usergrid.security.tokens.TokenService;
import org.apache.usergrid.services.ServiceManagerFactory;
@@ -77,7 +78,7 @@ public class ITSetup extends ExternalResource {
tomcatResource = TomcatResource.instance;
tomcatResource.setWebAppsPath( "src/main/webapp" );
- elasticSearchResource = ElasticSearchResource.instance;
+ elasticSearchResource = new ElasticSearchResource().startEs();
}
@@ -93,7 +94,6 @@ public class ITSetup extends ExternalResource {
synchronized ( cassandraResource ) {
super.before();
- elasticSearchResource.before();
emf = cassandraResource.getBean( EntityManagerFactory.class );
smf = cassandraResource.getBean( ServiceManagerFactory.class );
@@ -133,7 +133,7 @@ public class ITSetup extends ExternalResource {
protected void after() {
emf.flushEntityManagerCaches();
tomcatResource.after();
- elasticSearchResource.after();
+// elasticSearchResource.after();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java b/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
index 1d7aae0..788a4b3 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
@@ -16,12 +16,12 @@
*/
package org.apache.usergrid.rest;
-import org.apache.usergrid.ElasticSearchResource;
import org.junit.ClassRule;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.apache.usergrid.cassandra.CassandraResource;
import org.apache.usergrid.cassandra.Concurrent;
+import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.rest.applications.events.ApplicationRequestCounterIT;
import org.apache.usergrid.rest.applications.collection.devices.DevicesResourceIT;
import org.apache.usergrid.rest.applications.assets.AssetResourceIT;
@@ -57,7 +57,7 @@ import org.apache.usergrid.rest.management.organizations.AdminEmailEncodingIT;
public class RestITSuite {
@ClassRule
- public static ElasticSearchResource elasticSearchResource = ElasticSearchResource.instance;
+ public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource().startEs();
@ClassRule
public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
index 0b99cab..216e0a6 100644
--- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
@@ -29,6 +29,7 @@ import org.apache.usergrid.management.ApplicationCreator;
import org.apache.usergrid.management.ManagementService;
import org.apache.usergrid.management.export.ExportService;
import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.security.providers.SignInProviderFactory;
import org.apache.usergrid.security.tokens.TokenService;
import org.apache.usergrid.services.ServiceManagerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/25f7c6f4/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
index 4f2c151..d4e9751 100644
--- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
+++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
@@ -28,6 +28,7 @@ import org.apache.usergrid.management.RoleIT;
import org.apache.usergrid.management.cassandra.ApplicationCreatorIT;
import org.apache.usergrid.management.cassandra.ExportServiceIT;
import org.apache.usergrid.management.cassandra.ManagementServiceIT;
+import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.security.providers.FacebookProviderIT;
import org.apache.usergrid.security.providers.PingIdentityProviderIT;
import org.apache.usergrid.services.ActivitiesServiceIT;
@@ -59,5 +60,5 @@ public class ServiceITSuite {
public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts();
@ClassRule
- public static ElasticSearchResource elasticSearchResource = ElasticSearchResource.instance;
+ public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource().startEs();
}
[2/2] git commit: Fixes integer parsing bug in groovy script
Posted by to...@apache.org.
Fixes integer parsing bug in groovy script
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5a5377b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5a5377b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5a5377b3
Branch: refs/heads/es-client-changes
Commit: 5a5377b3868087b581e2ffd4fd087d56b75b2ca8
Parents: 25f7c6f
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 31 14:18:58 2014 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 31 14:18:58 2014 -0600
----------------------------------------------------------------------
stack/awscluster/src/main/groovy/configure_usergrid.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a5377b3/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index 65b82d0..bbbcd41 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -40,7 +40,7 @@ def clusterName = System.getenv().get("CASSANDRA_CLUSTER_NAME")
def superUserEmail = System.getenv().get("SUPER_USER_EMAIL")
def testAdminUserEmail = System.getenv().get("TEST_ADMIN_USER_EMAIL")
-def numEsNodes = System.getenv().get("ES_NUM_SERVERS")
+def numEsNodes = Integer.parseInt(System.getenv().get("ES_NUM_SERVERS"))
def esShards = numEsNodes/2;
def esReplicas = 2;