You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/03/12 00:05:53 UTC
[1/3] incubator-usergrid git commit: Fixed naming convention issue
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-466 ad33ecfd1 -> 77542c6b1
Fixed naming convention issue
Fixes blocking on cleanup (not necessary)
Fixes index creation when alias is cached
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c5943465
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c5943465
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c5943465
Branch: refs/heads/USERGRID-466
Commit: c594346558b9624c05f62096f8d83a0a91ce8af5
Parents: 9630fcf
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 17:02:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 17:02:58 2015 -0600
----------------------------------------------------------------------
.../results/FilteringLoader.java | 2 +-
.../index/impl/BufferQueueSQSImpl.java | 14 +-
.../index/impl/EsEntityIndexBatchImpl.java | 4 +-
.../index/impl/EsEntityIndexImpl.java | 14 +-
.../index/impl/EsIndexBufferConsumerImpl.java | 19 +--
.../persistence/index/impl/EsIndexCache.java | 138 +++++++++++--------
.../queue/impl/SQSQueueManagerImpl.java | 10 +-
7 files changed, 124 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index 1afd76b..2cd9fdc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -233,7 +233,7 @@ public class FilteringLoader implements ResultsLoader {
@Override
public void postProcess() {
- this.indexBatch.execute().get();
+ this.indexBatch.execute();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
index 25c2ba6..3cace11 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
@@ -127,6 +127,12 @@ public class BufferQueueSQSImpl implements BufferQueue {
@Override
public void offer( final IndexOperationMessage operation ) {
+ //no op
+ if(operation.isEmpty()){
+ operation.getFuture().done();
+ return;
+ }
+
final Timer.Context timer = this.writeTimer.time();
this.writeMeter.mark();
@@ -141,7 +147,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
//signal to SQS
this.queue.sendMessage( identifier );
- operation.getFuture().run();
+ operation.getFuture().done();
}
catch ( IOException e ) {
throw new RuntimeException( "Unable to queue message", e );
@@ -184,7 +190,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
}
//look up the values
- final Map<String, String> values = mapManager.getStrings( mapEntries );
+ final Map<String, String> storedCommands = mapManager.getStrings( mapEntries );
//load them into our response
@@ -193,7 +199,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
final String key = getMessageKey( message );
//now see if the key was there
- final String payload = values.get( key );
+ final String payload = storedCommands.get( key );
//the entry was not present in cassandra, ignore this message. Failure should eventually kick it to
// a DLQ
@@ -242,7 +248,7 @@ public class BufferQueueSQSImpl implements BufferQueue {
final SqsIndexOperationMessage sqsIndexOperationMessage = ( SqsIndexOperationMessage ) ioe;
- final String key = getMessageKey(sqsIndexOperationMessage.getMessage());
+ final String key = getMessageKey( sqsIndexOperationMessage.getMessage() );
//remove it from the map
mapManager.delete( key );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index b63dfe6..8481dab 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -197,7 +197,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
* No-op, just disregard it
*/
if(tempContainer.isEmpty()){
- return tempContainer.getFuture();
+ final BetterFuture<?> future = tempContainer.getFuture();
+ future.done();
+ return future;
}
return indexBatchBufferProducer.put(tempContainer);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 99643da..fa50734 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -209,7 +209,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public void initializeIndex() {
final int numberOfShards = config.getNumberOfShards();
final int numberOfReplicas = config.getNumberOfReplicas();
- String[] indexes = getIndexes(AliasType.Write);
+ String[] indexes = getIndexesFromEs(AliasType.Write);
if(indexes == null || indexes.length==0) {
addIndex(null, numberOfShards, numberOfReplicas, config.getWriteConsistencyLevel());
}
@@ -336,6 +336,18 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
/**
+ * Get our index info from ES, but clear our cache first
+ * @param aliasType
+ * @return
+ */
+ public String[] getIndexesFromEs(final AliasType aliasType){
+ aliasCache.invalidate( alias );
+ return getIndexes( aliasType );
+ }
+
+
+
+ /**
* Tests writing a document to a new index to ensure it's working correctly. See this post:
* http://s.apache.org/index-missing-exception
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 2762c18..862b1ae 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -46,6 +46,7 @@ import rx.Subscription;
import rx.functions.Action1;
import rx.functions.Action2;
import rx.functions.Func1;
+import rx.functions.Func2;
import rx.schedulers.Schedulers;
import java.util.List;
@@ -191,27 +192,27 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
}
//process and flatten all the messages to builder requests
- Observable<IndexOperationMessage> flattenMessages = Observable.from( operationMessages );
-
-
//batch shard operations into a bulk request
- flattenMessages.flatMap( new Func1<IndexOperationMessage, Observable<BatchRequest>>() {
+ Observable.from( operationMessages ).flatMap( new Func1<IndexOperationMessage, Observable<BatchRequest>>() {
@Override
public Observable<BatchRequest> call( final IndexOperationMessage indexOperationMessage ) {
final Observable<IndexRequest> index = Observable.from( indexOperationMessage.getIndexRequests() );
- final Observable<DeIndexRequest> deIndex = Observable.from( indexOperationMessage.getDeIndexRequests() );
+ final Observable<DeIndexRequest> deIndex =
+ Observable.from( indexOperationMessage.getDeIndexRequests() );
- indexSizeCounter.inc(indexOperationMessage.getDeIndexRequests().size());
- indexSizeCounter.inc(indexOperationMessage.getIndexRequests().size());
+ indexSizeCounter.dec( indexOperationMessage.getDeIndexRequests().size() );
+ indexSizeCounter.dec( indexOperationMessage.getIndexRequests().size() );
return Observable.merge( index, deIndex );
}
} )
//collection all the operations into a single stream
- .collect( initRequest(), new Action2<BulkRequestBuilder, BatchRequest>() {
+ .reduce( initRequest(), new Func2<BulkRequestBuilder, BatchRequest, BulkRequestBuilder>() {
@Override
- public void call( final BulkRequestBuilder bulkRequestBuilder, final BatchRequest batchRequest ) {
+ public BulkRequestBuilder call( final BulkRequestBuilder bulkRequestBuilder, final BatchRequest batchRequest ) {
batchRequest.doOperation( client, bulkRequestBuilder );
+
+ return bulkRequestBuilder;
}
} )
//send the request off to ES
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
index 0c07a34..ef518dd 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java
@@ -20,6 +20,24 @@
package org.apache.usergrid.persistence.index.impl;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.index.AliasedEntityIndex;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@@ -29,21 +47,6 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import org.apache.usergrid.persistence.index.AliasedEntityIndex;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexIdentifier;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.client.AdminClient;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
/**
@@ -52,65 +55,88 @@ import java.util.concurrent.TimeUnit;
@Singleton
public class EsIndexCache {
- private static final Logger logger = LoggerFactory.getLogger(EsEntityIndexImpl.class);
+ private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
private final ListeningScheduledExecutorService refreshExecutors;
private LoadingCache<String, String[]> aliasIndexCache;
+ private EsProvider provider;
+
@Inject
- public EsIndexCache(final EsProvider provider, final IndexFig indexFig) {
-
- this.refreshExecutors = MoreExecutors
- .listeningDecorator(Executors.newScheduledThreadPool(indexFig.getIndexCacheMaxWorkers()));
-
- aliasIndexCache = CacheBuilder.newBuilder().maximumSize(1000)
- .refreshAfterWrite(5,TimeUnit.MINUTES)
- .build(new CacheLoader<String, String[]>() {
- @Override
- public ListenableFuture<String[]> reload(final String key, String[] oldValue) throws Exception {
- ListenableFutureTask<String[]> task = ListenableFutureTask.create( new Callable<String[]>() {
- public String[] call() {
- return load( key );
- }
- } );
- refreshExecutors.execute(task);
- return task;
- }
-
- @Override
- public String[] load(final String aliasName) {
- final AdminClient adminClient = provider.getClient().admin();
- //remove write alias, can only have one
- ImmutableOpenMap<String, List<AliasMetaData>> aliasMap =
- adminClient.indices().getAliases(new GetAliasesRequest(aliasName)).actionGet().getAliases();
- return aliasMap.keys().toArray(String.class);
- }
- }) ;
+ public EsIndexCache( final EsProvider provider, final IndexFig indexFig ) {
+
+ this.refreshExecutors =
+ MoreExecutors.listeningDecorator( Executors.newScheduledThreadPool( indexFig.getIndexCacheMaxWorkers() ) );
+
+ this.provider = provider;
+
+ aliasIndexCache = CacheBuilder.newBuilder().maximumSize( 1000 ).refreshAfterWrite( 5, TimeUnit.MINUTES )
+ .build( new CacheLoader<String, String[]>() {
+ @Override
+ public ListenableFuture<String[]> reload( final String key,
+ String[] oldValue )
+ throws Exception {
+ ListenableFutureTask<String[]> task =
+ ListenableFutureTask.create( new Callable<String[]>() {
+ public String[] call() {
+ return load( key );
+ }
+ } );
+ refreshExecutors.execute( task );
+ return task;
+ }
+
+
+ @Override
+ public String[] load( final String aliasName ) {
+ return getIndexesFromEs(aliasName);
+ }
+ } );
}
-
+
/**
* Get indexes for an alias
*/
- public String[] getIndexes(IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType) {
+ public String[] getIndexes( IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
String[] indexes;
try {
- indexes = aliasIndexCache.get(aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias());
- } catch (ExecutionException ee) {
- logger.error("Failed to retreive indexes", ee);
- throw new RuntimeException(ee);
+ indexes = aliasIndexCache.get( getAliasName( alias, aliasType ) );
+ }
+ catch ( ExecutionException ee ) {
+ logger.error( "Failed to retreive indexes", ee );
+ throw new RuntimeException( ee );
}
return indexes;
}
-
+
+
+ private String[] getIndexesFromEs(final String aliasName){
+ final AdminClient adminClient = this.provider.getClient().admin();
+ //remove write alias, can only have one
+ ImmutableOpenMap<String, List<AliasMetaData>> aliasMap =
+ adminClient.indices().getAliases( new GetAliasesRequest( aliasName ) ).actionGet().getAliases();
+ return aliasMap.keys().toArray( String.class );
+ }
+
+
/**
- * clean up cache
+ * Get the name of the alias to use
+ * @param alias
+ * @param aliasType
+ * @return
*/
- public void invalidate(IndexIdentifier.IndexAlias alias){
- aliasIndexCache.invalidate(alias.getWriteAlias());
- aliasIndexCache.invalidate(alias.getReadAlias());
-
+ private String getAliasName( IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) {
+ return aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias();
}
+
+ /**
+ * clean up cache
+ */
+ public void invalidate( IndexIdentifier.IndexAlias alias ) {
+ aliasIndexCache.invalidate( alias.getWriteAlias() );
+ aliasIndexCache.invalidate( alias.getReadAlias() );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5943465/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
index a78fc80..1fbd9b6 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
@@ -33,7 +33,6 @@ import org.apache.usergrid.persistence.queue.QueueManager;
import org.apache.usergrid.persistence.queue.QueueMessage;
import org.apache.usergrid.persistence.queue.QueueScope;
-import com.amazonaws.AbortedException;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.sqs.AmazonSQSClient;
@@ -76,15 +75,16 @@ public class SQSQueueManagerImpl implements QueueManager {
.maximumSize( 1000 )
.build( new CacheLoader<String, Queue>() {
@Override
- public Queue load( String queueLoader ) throws Exception {
+ public Queue load( String queueName ) throws Exception {
//the amazon client is not thread safe, we need to create one per queue
Queue queue = null;
try {
- GetQueueUrlResult result = sqs.getQueueUrl( queueLoader );
+ GetQueueUrlResult result = sqs.getQueueUrl( queueName );
queue = new Queue( result.getQueueUrl() );
}catch ( QueueDoesNotExistException queueDoesNotExistException ) {
//no op, swallow
+ LOG.error( "Queue {} does not exist, creating", queueName );
}
catch ( Exception e ) {
@@ -92,7 +92,7 @@ public class SQSQueueManagerImpl implements QueueManager {
throw e;
}
if ( queue == null ) {
- CreateQueueRequest createQueueRequest = new CreateQueueRequest().withQueueName( queueLoader );
+ CreateQueueRequest createQueueRequest = new CreateQueueRequest().withQueueName( queueName );
CreateQueueResult result = sqs.createQueue( createQueueRequest );
String url = result.getQueueUrl();
queue = new Queue( url );
@@ -124,7 +124,7 @@ public class SQSQueueManagerImpl implements QueueManager {
private String getName() {
- String name = scope.getApplication().getType() + "_"+ scope.getName() + "_"+ scope.getApplication().getUuid().toString();
+ String name = fig.getPrefix() + "_" + scope.getApplication().getType() + "_"+ scope.getName() + "_"+ scope.getApplication().getUuid().toString();
return name;
}
[2/3] incubator-usergrid git commit: Changes nodes to 2 per shards
with 1 replica.
Posted by to...@apache.org.
Changes nodes to 2 per shards with 1 replica.
Changes ES system properties
Adds remote debugging to tomcat by default
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fa0705e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fa0705e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fa0705e7
Branch: refs/heads/USERGRID-466
Commit: fa0705e76b001be5e873cb0b1dd1c879d2e7b309
Parents: c594346
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 17:04:06 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 17:04:06 2015 -0600
----------------------------------------------------------------------
.../main/dist/init_instance/init_rest_server.sh | 20 +++++++++++---------
.../dist/init_instance/install_elasticsearch.sh | 10 ++++------
.../main/groovy/configure_elasticsearch.groovy | 11 +++++++++++
.../src/main/groovy/configure_usergrid.groovy | 2 +-
4 files changed, 27 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa0705e7/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
index cad4e74..ff74851 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
@@ -1,14 +1,14 @@
#!/bin/bash
-#
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. The ASF licenses this file to You
# under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -35,10 +35,10 @@ apt-get -y --force-yes install ${PKGS}
# Install AWS Java SDK and get it into the Groovy classpath
curl http://sdk-for-java.amazonwebservices.com/latest/aws-java-sdk.zip > /tmp/aws-sdk-java.zip
cd /usr/share/
-unzip /tmp/aws-sdk-java.zip
+unzip /tmp/aws-sdk-java.zip
mkdir -p /home/ubuntu/.groovy/lib
cp /usr/share/aws-java-sdk-*/third-party/*/*.jar /home/ubuntu/.groovy/lib
-cp /usr/share/aws-java-sdk-*/lib/* /home/ubuntu/.groovy/lib
+cp /usr/share/aws-java-sdk-*/lib/* /home/ubuntu/.groovy/lib
ln -s /home/ubuntu/.groovy /root/.groovy
# Build environment for Groovy scripts
@@ -55,7 +55,7 @@ groovy tag_instance.groovy -BUILD-IN-PROGRESS
chmod +x /usr/share/usergrid/update.sh
cd /usr/share/usergrid/init_instance
-./install_oraclejdk.sh
+./install_oraclejdk.sh
cd /usr/share/usergrid/init_instance
./install_yourkit.sh
@@ -119,6 +119,8 @@ sed -i.bak "s/<Connector/<Connector maxThreads=\"${TOMCAT_THREADS}\" acceptCount
#Append our java opts for secret key
echo "JAVA_OPTS=\"\${JAVA_OPTS} -DAWS_SECRET_KEY=${AWS_SECRET_KEY} -DAWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY}\"" >> /etc/default/tomcat7
+echo "JAVA_OPTS=\"\${JAVA_OPTS} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000\"" >> /etc/default/tomcat7
+
ulimit -n $NOFILE
@@ -168,7 +170,7 @@ kernel.shmall = 4294967296
######
EOF
-# wait for enough Cassandra nodes then delpoy and configure Usergrid
+# wait for enough Cassandra nodes then delpoy and configure Usergrid
cd /usr/share/usergrid/scripts
groovy wait_for_instances.groovy cassandra ${CASSANDRA_NUM_SERVERS}
groovy wait_for_instances.groovy elasticsearch ${ES_NUM_SERVERS}
@@ -182,8 +184,8 @@ chown -R tomcat7 /usr/share/usergrid/webapps
chown -R tomcat7 /var/lib/tomcat7/webapps
# configure usergrid
-mkdir -p /usr/share/tomcat7/lib
-groovy configure_usergrid.groovy > /usr/share/tomcat7/lib/usergrid-deployment.properties
+mkdir -p /usr/share/tomcat7/lib
+groovy configure_usergrid.groovy > /usr/share/tomcat7/lib/usergrid-deployment.properties
groovy configure_portal_new.groovy >> /var/lib/tomcat7/webapps/portal/config.js
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa0705e7/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
index b2018e1..b488320 100644
--- a/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
+++ b/stack/awscluster/src/main/dist/init_instance/install_elasticsearch.sh
@@ -80,8 +80,10 @@ cat >> /etc/default/elasticsearch << EOF
ES_HEAP_SIZE=${ES_HEAP_SIZE}
MAX_OPEN_FILES=65535
MAX_MAP_COUNT=262144
-#MAX_LOCKED_MEMORY=unlimited
+MAX_LOCKED_MEMORY=unlimited
JAVA_HOME=/usr/lib/jvm/jdk1.7.0
+ES_HEAP_NEWSIZE=4g
+ES_JAVA_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:SurvivorRatio=4 -Xloggc:/mnt/raid/elasticsearch/jvm"
EOF
#Set it because Matt says so
@@ -106,15 +108,11 @@ pushd /usr/share/elasticsearch/bin
#Install bigdesk
-./plugin --install lukas-vlcek/bigdesk
-
-./plugin --install mobz/elasticsearch-head
-
./plugin -install royrusso/elasticsearch-HQ
./plugin -install karmi/elasticsearch-paramedic
-./plugin -install xyu/elasticsearch-whatson/0.1.3
+./plugin -install elasticsearch/elasticsearch-cloud-aws/2.4.1
popd
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa0705e7/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
index 19c969d..38b032f 100644
--- a/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
+++ b/stack/awscluster/src/main/groovy/configure_elasticsearch.groovy
@@ -26,6 +26,9 @@
//
+String accessKey = (String)System.getenv().get("AWS_ACCESS_KEY")
+String secretKey = (String)System.getenv().get("AWS_SECRET_KEY")
+
String hostName = (String)System.getenv().get("PUBLIC_HOSTNAME")
def clusterName = (String)System.getenv().get("ES_CLUSTER_NAME")
@@ -153,6 +156,14 @@ index.indexing.slowlog.threshold.index.info: 5s
index.indexing.slowlog.threshold.index.debug: 2s
index.indexing.slowlog.threshold.index.trace: 500ms
+########
+# AWS PLUGIM
+##########
+
+cloud.aws.access_key: ${accessKey}
+cloud.aws.secret_key: ${secretKey}
+
+
"""
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fa0705e7/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index 9274430..8af5884 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -44,7 +44,7 @@ def testAdminUserEmail = System.getenv().get("TEST_ADMIN_USER_EMAIL")
def numEsNodes = Integer.parseInt(System.getenv().get("ES_NUM_SERVERS"))
//Override number of shards. Set it to 2x the cluster size
-def esShards = numEsNodes;
+def esShards = numEsNodes*2;
//This gives us 3 copies, which means we'll have a quorum with primary + 1 replica
[3/3] incubator-usergrid git commit: Merge branch 'USERGRID-466' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Posted by to...@apache.org.
Merge branch 'USERGRID-466' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-466
Conflicts:
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/77542c6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/77542c6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/77542c6b
Branch: refs/heads/USERGRID-466
Commit: 77542c6b1bd96f402b33857b453431689ec5b085
Parents: fa0705e ad33ecf
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 11 17:05:47 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 11 17:05:47 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/IndexOperationMessage.java | 4 ++++
.../usergrid/persistence/index/impl/BufferQueueInMemoryImpl.java | 1 +
.../usergrid/persistence/index/impl/BufferQueueSQSImpl.java | 2 +-
3 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/77542c6b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/BufferQueueSQSImpl.java
----------------------------------------------------------------------