You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2016/08/22 15:00:42 UTC
[38/50] usergrid git commit: Fix bootstrapping issue and limit the
amount of cache used for Graph Node Shards.
Fix bootstrapping issue and limit the amount of cache used for Graph Node Shards.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ceb50ff4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ceb50ff4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ceb50ff4
Branch: refs/heads/asf-site
Commit: ceb50ff458f9e2cee8e6cbc6a3a4eed1f95bf638
Parents: b1d3e25
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Aug 18 17:49:38 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Aug 18 17:49:38 2016 -0700
----------------------------------------------------------------------
.../exception/CollectionRuntimeException.java | 2 +-
.../data/MigrationInfoSerializationImpl.java | 4 +-
.../core/migration/util/AstyanaxUtils.java | 50 ++++++++++++++++++++
.../usergrid/persistence/graph/GraphFig.java | 2 +-
.../impl/shard/impl/NodeShardCacheImpl.java | 28 ++---------
5 files changed, 57 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ceb50ff4/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
index 8aa2a7a..a6457cf 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
@@ -61,7 +61,7 @@ public class CollectionRuntimeException extends RuntimeException {
if ( getCause() instanceof BadRequestException ) {
BadRequestException bre = (BadRequestException)getCause();
String msg = bre.getMessage();
- if ( msg.contains("Keyspace") && msg.contains( "does not exist" ) ) {
+ if ( (msg.contains("Keyspace") && msg.contains( "does not exist" )) || msg.contains("unconfigured columnfamily")) {
return true;
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ceb50ff4/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
index 18427f7..ee04293 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
@@ -28,7 +28,7 @@ import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.usergrid.persistence.core.astyanax.*;
import org.apache.usergrid.persistence.core.datastax.TableDefinition;
-import org.apache.usergrid.persistence.core.migration.util.AstayanxUtils;
+import org.apache.usergrid.persistence.core.migration.util.AstyanaxUtils;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
@@ -143,7 +143,7 @@ public class MigrationInfoSerializationImpl implements MigrationInfoSerializatio
return 0;
}
catch ( ConnectionException e ) {
- AstayanxUtils.isKeyspaceMissing("Unable to connect to cassandra to retrieve status", e);
+ AstyanaxUtils.isSchemaMissing("Unable to connect to cassandra to retrieve status", e);
return 0;
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ceb50ff4/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/util/AstyanaxUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/util/AstyanaxUtils.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/util/AstyanaxUtils.java
new file mode 100644
index 0000000..e83fedf
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/util/AstyanaxUtils.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.migration.util;
+
+
+import com.netflix.astyanax.connectionpool.exceptions.BadRequestException;
+
+
+public class AstyanaxUtils {
+
+ /**
+ * Return true if the exception is an instance of a missing keysapce
+ * @param rethrowMessage The message to add to the exception if rethrown
+ * @param cassandraException The exception from cassandar
+ * @return
+ */
+ public static void isSchemaMissing(final String rethrowMessage, final Exception cassandraException ) {
+
+ if ( cassandraException instanceof BadRequestException ) {
+
+ //check if it's b/c the keyspace is missing, if so
+ final String message = cassandraException.getMessage();
+
+ //no op, just swallow
+ if( (message.contains( "why:Keyspace" ) && message.contains( "does not exist" ))
+ || message.contains("why:unconfigured columnfamily")){
+ return;
+ };
+ }
+
+ throw new RuntimeException( rethrowMessage, cassandraException );
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ceb50ff4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
index efd94ed..46f6f0c 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
@@ -134,7 +134,7 @@ public interface GraphFig extends GuicyFig {
long getShardMinDelta();
- @Default("250000")
+ @Default("100000")
@Key(SHARD_CACHE_SIZE)
long getShardCacheSize();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ceb50ff4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
index 545ac37..ee9602c 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
@@ -72,18 +72,8 @@ import com.google.inject.Inject;
@Singleton
public class NodeShardCacheImpl implements NodeShardCache {
- /**
- * Only cache shards that have < 10k groups. This is an arbitrary amount, and may change with profiling and
- * testing
- */
- private static final int MAX_WEIGHT_PER_ELEMENT = 10000;
-
-
private final NodeShardAllocation nodeShardAllocation;
private final GraphFig graphFig;
-
-
-
private ListeningScheduledExecutorService refreshExecutors;
private LoadingCache<CacheKey, CacheEntry> graphs;
@@ -215,10 +205,10 @@ public class NodeShardCacheImpl implements NodeShardCache {
//wait for a trip to cassandra
.refreshAfterWrite( graphFig.getShardCacheTimeout(), TimeUnit.MILLISECONDS )
- //set our weight function, since not all shards are equal
- .maximumWeight(MAX_WEIGHT_PER_ELEMENT * graphFig.getShardCacheSize() ).weigher( new ShardWeigher() )
+ //set a static cache entry size here
+ .maximumSize(graphFig.getShardCacheSize())
- //set our shard loader
+ //set our shard loader
.build( new ShardCacheLoader() );
}
@@ -363,16 +353,4 @@ public class NodeShardCacheImpl implements NodeShardCache {
//TODO, use RX for sliding window buffering and duplicate removal
}
-
-
- /**
- * Calculates the weight of the entry by geting the size of the cache
- */
- final class ShardWeigher implements Weigher<CacheKey, CacheEntry> {
-
- @Override
- public int weigh( final CacheKey key, final CacheEntry value ) {
- return value.getCacheSize();
- }
- }
}