You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ja...@apache.org on 2014/08/07 20:54:24 UTC
[1/3] git commit: Workaround for netty issue causing corrupted data
to come off the wire
Repository: cassandra
Updated Branches:
refs/heads/trunk 32358d646 -> 4c3ba4f30
Workaround for netty issue causing corrupted data to come off the wire
patch by tjake, test by Johan Bjork; reviewed by belliottsmith for (CASSANDRA-7695)
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b3ada2bc
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b3ada2bc
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b3ada2bc
Branch: refs/heads/trunk
Commit: b3ada2bc453e84a470b070b56e35a13e0913662b
Parents: d8eff03
Author: Jake Luciani <ja...@apache.org>
Authored: Thu Aug 7 14:35:28 2014 -0400
Committer: Jake Luciani <ja...@apache.org>
Committed: Thu Aug 7 14:35:28 2014 -0400
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/service/CassandraDaemon.java | 9 +
.../apache/cassandra/cql3/CorruptionTest.java | 195 +++++++++++++++++++
3 files changed, 205 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b3ada2bc/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 26b39e0..aef0c40 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.0-final
+ * workaround for netty issue causing corrupted data off the wire (CASSANDRA-7695)
* cqlsh DESC CLUSTER fails retrieving ring information (CASSANDRA-7687)
* Fix binding null values inside UDT (CASSANDRA-7685)
* Fix UDT field selection with empty fields (CASSANDRA-7670)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b3ada2bc/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java
index 7c85f81..5c88cb1 100644
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@ -37,6 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.addthis.metrics.reporter.config.ReporterConfig;
+import io.netty.util.internal.PlatformDependent;
import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
import org.apache.cassandra.concurrent.Stage;
import org.apache.cassandra.concurrent.StageManager;
@@ -69,6 +70,14 @@ import org.apache.cassandra.utils.Pair;
*/
public class CassandraDaemon
{
+
+ //Workaround for netty issue
+ static
+ {
+ System.setProperty("io.netty.noUnsafe","true");
+ assert !PlatformDependent.hasUnsafe();
+ }
+
public static final String MBEAN_NAME = "org.apache.cassandra.db:type=NativeAccess";
// Have a dedicated thread to call exit to avoid deadlock in the case where the thread that wants to invoke exit
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b3ada2bc/test/long/org/apache/cassandra/cql3/CorruptionTest.java
----------------------------------------------------------------------
diff --git a/test/long/org/apache/cassandra/cql3/CorruptionTest.java b/test/long/org/apache/cassandra/cql3/CorruptionTest.java
new file mode 100644
index 0000000..1a42112
--- /dev/null
+++ b/test/long/org/apache/cassandra/cql3/CorruptionTest.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cql3;
+
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.datastax.driver.core.*;
+import com.datastax.driver.core.policies.LoggingRetryPolicy;
+import com.datastax.driver.core.policies.Policies;
+import com.datastax.driver.core.utils.Bytes;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.service.EmbeddedCassandraService;
+
+public class CorruptionTest extends SchemaLoader
+{
+
+ private static EmbeddedCassandraService cassandra;
+ private static Cluster cluster;
+ private static Session session;
+
+ private static PreparedStatement getStatement;
+ private static PreparedStatement putStatement;
+ private static String KEYSPACE = "cass_test";
+ private static final String TABLE="put_test";
+ private static final String KEY = "SingleFailingKey";
+ private static String VALUE;
+ private final int THREADPOOL_SIZE=40;
+
+ @BeforeClass()
+ public static void setup() throws ConfigurationException, IOException
+ {
+ Schema.instance.clear();
+
+ cassandra = new EmbeddedCassandraService();
+ cassandra.start();
+
+ cluster = Cluster.builder().addContactPoint("127.0.0.1")
+ .withRetryPolicy(new LoggingRetryPolicy(Policies.defaultRetryPolicy()))
+ .withPort(DatabaseDescriptor.getNativeTransportPort()).build();
+ session = cluster.connect();
+
+ session.execute("CREATE KEYSPACE IF NOT EXISTS " + KEYSPACE +" WITH replication " +
+ "= {'class':'SimpleStrategy', 'replication_factor':1};");
+ session.execute("USE " + KEYSPACE);
+ session.execute("CREATE TABLE IF NOT EXISTS " + TABLE + " (" +
+ "key blob," +
+ "value blob," +
+ "PRIMARY KEY (key));");
+
+
+ // Prepared statements
+ getStatement = session.prepare("SELECT value FROM " + TABLE + " WHERE key = ?;");
+ getStatement.setConsistencyLevel(ConsistencyLevel.LOCAL_QUORUM);
+
+ putStatement = session.prepare("INSERT INTO " + TABLE + " (key, value) VALUES (?, ?);");
+ putStatement.setConsistencyLevel(ConsistencyLevel.LOCAL_QUORUM);
+
+
+
+ StringBuilder s = new StringBuilder();
+ char a='a';
+ char z='z';
+ for (int i = 0; i < 500*1024; i++)
+ {
+ char x = (char)((i%((z-a)+1))+a);
+ if (x == 'a')
+ {
+ x = '\n';
+ }
+ s.append(x);
+ }
+ VALUE = s.toString();
+ }
+
+ @Test
+ public void runCorruptionTest()
+ {
+
+ final CountDownLatch failure = new CountDownLatch(1);
+
+
+ ExecutorService executor = Executors.newFixedThreadPool(THREADPOOL_SIZE);
+ for (int i = 0; i < THREADPOOL_SIZE; i++)
+ {
+ executor.execute(new Runnable()
+ {
+ @Override
+ public void run()
+ {
+ for (int i = 0; i < 100000; i++)
+ {
+ put(KEY.getBytes(), VALUE.getBytes());
+ byte[] res = get(KEY.getBytes());
+ //since we're flooding the server we might get some timeouts, that's not
+ //relevant for this test
+ if (res == null)
+ continue;
+
+ if (!Arrays.equals(VALUE.getBytes(), res))
+ {
+ /*try
+ {
+ dumpKeys(VALUE.getBytes(), res);
+ }
+ catch (IOException e)
+ {
+ e.printStackTrace();
+ }*/
+ failure.countDown();
+ }
+ }
+ }
+
+ private void dumpKeys(byte[] putdata, byte[] getdata) throws IOException {
+ String basename = "bad-data-tid" + Thread.currentThread().getId();
+ File put = new File(basename+"-put");
+ File get = new File(basename+"-get");
+ try(FileWriter pw = new FileWriter(put)) {
+ pw.write(new String(putdata));
+ }
+ try(FileWriter pw = new FileWriter(get)) {
+ pw.write(new String(getdata));
+ }
+ }
+ });
+ }
+
+ try
+ {
+ assert!failure.await(2, TimeUnit.MINUTES);
+ }
+ catch (InterruptedException e)
+ {
+
+ }
+ executor.shutdownNow();
+
+ }
+
+ public static byte[] get(byte[] key)
+ {
+ BoundStatement boundStatement = new BoundStatement(getStatement);
+ boundStatement.setBytes(0, ByteBuffer.wrap(key));
+
+ final com.datastax.driver.core.ResultSet resultSet = session.execute(boundStatement);
+ final Row row = resultSet.one();
+ if (row != null)
+ {
+ final ByteBuffer byteBuf = row.getBytes("value");
+ return Bytes.getArray(byteBuf);
+ }
+
+ return null;
+ }
+
+ public static void put(byte[] key, byte[] value)
+ {
+ BoundStatement boundStatement = new BoundStatement(putStatement);
+ boundStatement.setBytes(0, ByteBuffer.wrap(key));
+ boundStatement.setBytes(1, ByteBuffer.wrap(value));
+
+ session.execute(boundStatement);
+ }
+}
[3/3] git commit: Merge branch 'cassandra-2.1' into trunk
Posted by ja...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/4c3ba4f3
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/4c3ba4f3
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/4c3ba4f3
Branch: refs/heads/trunk
Commit: 4c3ba4f30acf65d769517009f8d263f6d87ede81
Parents: 32358d6 2c701f8
Author: Jake Luciani <ja...@apache.org>
Authored: Thu Aug 7 14:53:30 2014 -0400
Committer: Jake Luciani <ja...@apache.org>
Committed: Thu Aug 7 14:53:30 2014 -0400
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/service/CassandraDaemon.java | 9 +
.../apache/cassandra/cql3/CorruptionTest.java | 195 +++++++++++++++++++
3 files changed, 205 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/4c3ba4f3/CHANGES.txt
----------------------------------------------------------------------
[2/3] git commit: Merge branch 'cassandra-2.1.0' into cassandra-2.1
Posted by ja...@apache.org.
Merge branch 'cassandra-2.1.0' into cassandra-2.1
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2c701f8f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2c701f8f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2c701f8f
Branch: refs/heads/trunk
Commit: 2c701f8f9a87bd4388e917eab8910f7fa865b5bc
Parents: 756c85e b3ada2b
Author: Jake Luciani <ja...@apache.org>
Authored: Thu Aug 7 14:52:56 2014 -0400
Committer: Jake Luciani <ja...@apache.org>
Committed: Thu Aug 7 14:52:56 2014 -0400
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/service/CassandraDaemon.java | 9 +
.../apache/cassandra/cql3/CorruptionTest.java | 195 +++++++++++++++++++
3 files changed, 205 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2c701f8f/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index c7f2510,aef0c40..2bd8667
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,25 -1,5 +1,26 @@@
+2.1.1
+ * Avoid IOOBE when building SyntaxError message snippet (CASSANDRA-7569)
+ * SSTableExport uses correct validator to create string representation of partition
+ keys (CASSANDRA-7498)
+ * Avoid NPEs when receiving type changes for an unknown keyspace (CASSANDRA-7689)
+ * Add support for custom 2i validation (CASSANDRA-7575)
+ * Pig support for hadoop CqlInputFormat (CASSANDRA-6454)
+ * Add listen_interface and rpc_interface options (CASSANDRA-7417)
+ * Improve schema merge performance (CASSANDRA-7444)
+ * Adjust MT depth based on # of partition validating (CASSANDRA-5263)
+ * Optimise NativeCell comparisons (CASSANDRA-6755)
+ * Configurable client timeout for cqlsh (CASSANDRA-7516)
+ * Include snippet of CQL query near syntax error in messages (CASSANDRA-7111)
+Merged from 2.0:
+ * Fix IncompatibleClassChangeError from hadoop2 (CASSANDRA-7229)
+ * Add 'nodetool sethintedhandoffthrottlekb' (CASSANDRA-7635)
+ * (cqlsh) Add tab-completion for CREATE/DROP USER IF [NOT] EXISTS (CASSANDRA-7611)
+ * Catch errors when the JVM pulls the rug out from GCInspector (CASSANDRA-5345)
+ * cqlsh fails when version number parts are not int (CASSANDRA-7524)
+
+
2.1.0-final
+ * workaround for netty issue causing corrupted data off the wire (CASSANDRA-7695)
* cqlsh DESC CLUSTER fails retrieving ring information (CASSANDRA-7687)
* Fix binding null values inside UDT (CASSANDRA-7685)
* Fix UDT field selection with empty fields (CASSANDRA-7670)