You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/11/19 01:20:34 UTC
[4/9] incubator-usergrid git commit: Adds DataCorruptionException to
detect parse failure and return an empty entity,
since it can no longer be read and should be deleted.
Adds DataCorruptionException to detect parse failure and return an empty entity, since it can no longer be read and should be deleted.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b1aa75e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b1aa75e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b1aa75e8
Branch: refs/heads/two-dot-o-events
Commit: b1aa75e8d40900f4616c55735598aa6ce772c324
Parents: 1f0c8d7
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 17 19:06:32 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 17 19:06:32 2014 -0700
----------------------------------------------------------------------
.../persistence/PerformanceEntityWriteTest.java | 2 +-
.../exception/DataCorruptionException.java | 32 +++++++++++++++
.../MvccEntitySerializationStrategyImpl.java | 43 ++++++++++++++++----
.../graph/GraphManagerShardConsistencyIT.java | 2 +-
4 files changed, 68 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b1aa75e8/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java
index 9dfa3c4..64342a1 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java
@@ -147,7 +147,7 @@ public class PerformanceEntityWriteTest extends AbstractCoreIT {
addToCollectionEntity.put( "key2", 2000 );
addToCollectionEntity.put( "key3", "Some value" );
- final List<EntityRef> owners = Arrays.asList( ( EntityRef ) owner1, ( EntityRef ) owner2 );
+ final List<EntityRef> owners = Arrays.<EntityRef>asList( owner1, owner2 );
int i = 0;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b1aa75e8/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/DataCorruptionException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/DataCorruptionException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/DataCorruptionException.java
new file mode 100644
index 0000000..3089e38
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/DataCorruptionException.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.exception;
+
+
+/**
+ * Exception thrown when we are unable to parse data. This seems to be a data corruption issue with
+ * Abstract composite creation
+ */
+public class DataCorruptionException extends RuntimeException{
+
+ public DataCorruptionException( final String message, final Throwable cause ) {
+ super( message, cause );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b1aa75e8/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
index dc12087..b97429b 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
@@ -38,6 +38,7 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException;
+import org.apache.usergrid.persistence.collection.exception.DataCorruptionException;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.serialization.EntityRepair;
@@ -394,21 +395,34 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
@Override
public MvccEntity parseColumn( Column<UUID> column ) {
- final EntityWrapper deSerialized = column.getValue( ENTITY_JSON_SER );
+ final EntityWrapper deSerialized;
+ final UUID version = column.getName();
+
+ try {
+ deSerialized = column.getValue( ENTITY_JSON_SER );
+ }
+ catch ( DataCorruptionException e ) {
+ log.error(
+ "DATA CORRUPTION DETECTED when de-serializing entity with Id {} and version {}. This means the"
+ + " write was truncated.",
+ id, version );
+ //return an empty entity, we can never load this one, and we don't want it to bring the system
+ //to a grinding halt
+ return new MvccEntityImpl( id, version, MvccEntity.Status.DELETED, Optional.<Entity>absent() );
+ }
//Inject the id into it.
if ( deSerialized.entity.isPresent() ) {
EntityUtils.setId( deSerialized.entity.get(), id );
}
- return new MvccEntityImpl( id, column.getName(), deSerialized.status, deSerialized.entity );
+ return new MvccEntityImpl( id, version, deSerialized.status, deSerialized.entity );
}
}
public static class EntitySerializer extends AbstractSerializer<EntityWrapper> {
- public static final EntitySerializer INSTANCE = new EntitySerializer();
public static final SmileFactory f = new SmileFactory();
@@ -421,9 +435,6 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
private static byte[] VERSION = new byte[] { 0 };
- //the marker for when we're passed a "null" value
- private static final byte[] EMPTY = new byte[] { 0x0 };
-
public EntitySerializer() {
try {
@@ -479,7 +490,21 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
@Override
public EntityWrapper fromByteBuffer( final ByteBuffer byteBuffer ) {
- CompositeParser parser = Composites.newCompositeParser( byteBuffer );
+
+ /**
+ * We intentionally turn data corruption exceptions when we're unable to de-serialize
+ * the data in cassandra. If this occurs, we'll never be able to de-serialize it
+ * and it should be considered lost. This is an error that is occuring due to a bug
+ * in serializing the entity. This is a lazy recognition + repair signal for deployment with
+ * existing systems.
+ */
+ CompositeParser parser;
+ try {
+ parser = Composites.newCompositeParser( byteBuffer );
+ }
+ catch ( Exception e ) {
+ throw new DataCorruptionException("Unable to de-serialze entity", e);
+ }
byte[] version = parser.read( BYTES_ARRAY_SERIALIZER );
@@ -495,7 +520,7 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
return new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() );
}
- Entity storedEntity = null;
+ Entity storedEntity;
ByteBuffer jsonBytes = parser.read( BUFFER_SERIALIZER );
byte[] array = jsonBytes.array();
@@ -506,7 +531,7 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
storedEntity = mapper.readValue( array, start, length, Entity.class );
}
catch ( Exception e ) {
- throw new RuntimeException( "Unable to read entity data", e );
+ throw new DataCorruptionException( "Unable to read entity data", e );
}
final Optional<Entity> entity = Optional.of( storedEntity );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b1aa75e8/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
index 0f25537..095f855 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
@@ -79,7 +79,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
-@Ignore( "Kills cassandra, needs to be part of functional testing" )
+//@Ignore( "Kills cassandra, needs to be part of functional testing" )
public class GraphManagerShardConsistencyIT {
private static final Logger log = LoggerFactory.getLogger( GraphManagerShardConsistencyIT.class );