You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/03/03 20:13:20 UTC
[1/2] incubator-usergrid git commit: add write consistency
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-432 44c098942 -> 2b3101581
add write consistency
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5e578d5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5e578d5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5e578d5c
Branch: refs/heads/USERGRID-432
Commit: 5e578d5c7a85df521968678b3fa9df1d4241c065
Parents: 44c0989
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Mar 3 10:34:55 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Mar 3 10:34:55 2015 -0700
----------------------------------------------------------------------
.../usergrid/corepersistence/CpEntityManagerFactory.java | 4 ++--
.../org/apache/usergrid/persistence/EntityManagerFactory.java | 3 ++-
.../persistence/cassandra/EntityManagerFactoryImpl.java | 2 +-
.../org/apache/usergrid/persistence/index/EntityIndex.java | 3 ++-
.../usergrid/persistence/index/impl/EsEntityIndexImpl.java | 7 ++++---
.../usergrid/persistence/index/impl/EntityIndexTest.java | 4 ++--
.../src/main/java/org/apache/usergrid/rest/IndexResource.java | 4 +++-
7 files changed, 16 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e578d5c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 8272d7d..84fad36 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -826,9 +826,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
@Override
- public void addIndex(final UUID applicationId,final String indexSuffix,final int shards,final int replicas){
+ public void addIndex(final UUID applicationId,final String indexSuffix,final int shards,final int replicas, final String writeConsistency){
EntityIndex entityIndex = managerCache.getEntityIndex(CpNamingUtils.getApplicationScope(applicationId));
- entityIndex.addIndex(indexSuffix, shards, replicas);
+ entityIndex.addIndex(indexSuffix, shards, replicas,writeConsistency);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e578d5c/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index b668e24..880fe89 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -165,8 +165,9 @@ public interface EntityManagerFactory {
* @param suffix unique indentifier for additional index
* @param shards number of shards
* @param replicas number of replicas
+ * @param writeConsistency only "one, quorum, or all"
*/
- public void addIndex(final UUID appId,final String suffix,final int shards,final int replicas);
+ public void addIndex(final UUID appId,final String suffix,final int shards,final int replicas, final String writeConsistency);
public Health getEntityStoreHealth();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e578d5c/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 9a90e59..af1eabc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -466,7 +466,7 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
}
@Override
- public void addIndex(UUID appId, String suffix,final int shards,final int replicas) {
+ public void addIndex(UUID appId, String suffix,final int shards,final int replicas,final String consistency) {
throw new UnsupportedOperationException("Not supported in v1");
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e578d5c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index b888e09..3edde4d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -52,8 +52,9 @@ public interface EntityIndex {
* @param indexSuffix index name
* @param shards
* @param replicas
+ * @param writeConsistency
*/
- public void addIndex(final String indexSuffix, final int shards, final int replicas);
+ public void addIndex(final String indexSuffix, final int shards, final int replicas, final String writeConsistency);
/**
* Create the index batch.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e578d5c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index a8b0cc4..1910cdc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -140,11 +140,11 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public void initializeIndex() {
final int numberOfShards = config.getNumberOfShards();
final int numberOfReplicas = config.getNumberOfReplicas();
- addIndex(null, numberOfShards, numberOfReplicas);
+ addIndex(null, numberOfShards, numberOfReplicas,config.getWriteConsistencyLevel());
}
@Override
- public void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas) {
+ public void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas, final String writeConsistency) {
String normalizedSuffix = StringUtils.isNotEmpty(indexSuffix) ? indexSuffix : null;
try {
@@ -161,7 +161,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
Settings settings = ImmutableSettings.settingsBuilder()
.put("index.number_of_shards", numberOfShards)
.put("index.number_of_replicas", numberOfReplicas)
- .build();
+ .put("action.write_consistency", writeConsistency)
+ .build();
final CreateIndexResponse cir = admin.indices().prepareCreate(indexName)
.setSettings(settings)
.execute()
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e578d5c/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 7bfbb52..22aadc0 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -169,7 +169,7 @@ public class EntityIndexTest extends BaseIT {
testQueries( indexScope, searchTypes, entityIndex );
- entityIndex.addIndex("v2", 1,0);
+ entityIndex.addIndex("v2", 1,0,"one");
insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",101,100);
@@ -199,7 +199,7 @@ public class EntityIndexTest extends BaseIT {
entityIndex.refresh();
- entityIndex.addIndex("v2", 1, 0);
+ entityIndex.addIndex("v2", 1, 0, "one");
insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json", 1, 0);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e578d5c/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
index f5338c5..f4ebf23 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
@@ -250,8 +250,10 @@ public class IndexResource extends AbstractContextResource {
throw new IllegalArgumentException("Please add an indexSuffix to your post");
}
+ String writeConsistency = config.get("writeConsistency") != null ? (String)config.get("writeConsistency") : "one" ;
+
emf.addIndex(appId, config.get("indexSuffix").toString(),
- (int) config.get("shards"),(int) config.get("replicas"));
+ (int) config.get("shards"),(int) config.get("replicas"),writeConsistency);
response.setAction("Add index to alias");
return new JSONWithPadding(response, callback);
[2/2] incubator-usergrid git commit: adding graphite metrics
Posted by sf...@apache.org.
adding graphite metrics
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2b310158
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2b310158
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2b310158
Branch: refs/heads/USERGRID-432
Commit: 2b31015811205695024eb4b249dab35d4732a286
Parents: 5e578d5
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Mar 3 12:13:12 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Mar 3 12:13:12 2015 -0700
----------------------------------------------------------------------
.../usergrid/corepersistence/StaleIndexCleanupTest.java | 2 +-
.../persistence/index/impl/EsIndexBufferConsumerImpl.java | 7 ++++++-
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2b310158/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 703a497..54f0e93 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -385,7 +385,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
do {
if(count>0){Thread.sleep(200);}
crs = queryCollectionCp("things", "thing", "select *");
- } while ( crs.size() == numEntities && count++ < 15 );
+ } while ( crs.size() == numEntities && count++ < 1000 );
Assert.assertEquals("Expect candidates without earlier stale entities", crs.size(),numEntities);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2b310158/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index fbb7a40..52d1abb 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.persistence.index.impl;
import com.codahale.metrics.Counter;
+import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -61,10 +62,12 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
private final Observable<List<IndexOperationMessage>> consumer;
private final Timer flushTimer;
private final Counter indexSizeCounter;
+ private final Meter flushMeter;
@Inject
public EsIndexBufferConsumerImpl(final IndexFig config, final IndexBufferProducer producer, final EsProvider provider, final MetricsFactory metricsFactory){
this.flushTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class, "index.buffer.flush");
+ this.flushMeter = metricsFactory.getMeter(EsIndexBufferConsumerImpl.class, "index.buffer.meter");
this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferConsumerImpl.class, "index.buffer.size");
this.config = config;
this.failureMonitor = new FailureMonitorImpl(config,provider);
@@ -77,9 +80,11 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
.doOnNext(new Action1<List<IndexOperationMessage>>() {
@Override
public void call(List<IndexOperationMessage> containerList) {
- flushTimer.time();
if (containerList.size() > 0) {
+ flushMeter.mark(containerList.size());
+ Timer.Context time = flushTimer.time();
execute(containerList);
+ time.stop();
}
}
});