You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2013/08/08 01:05:28 UTC
[01/12] git commit: fix HsHa to respect max frame size patch by Pavel
Yaskevich; reviewed by Tyler Hobbs (CASSANDRA-4573)
Updated Branches:
refs/heads/cassandra-1.2 27943b117 -> 7d45d2bde
refs/heads/cassandra-2.0 18f0234bf -> 23becceb7
refs/heads/cassandra-2.0.0 19767fcd1 -> 88fea9fe7
refs/heads/trunk 9cf1068b7 -> 66182082c
fix HsHa to respect max frame size
patch by Pavel Yaskevich; reviewed by Tyler Hobbs (CASSANDRA-4573)
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/19767fcd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/19767fcd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/19767fcd
Branch: refs/heads/cassandra-2.0
Commit: 19767fcd173418002e82a10e1b08a2231bb29e99
Parents: af85f23
Author: Pavel Yaskevich <xe...@apache.org>
Authored: Tue Aug 6 15:20:48 2013 -0700
Committer: Pavel Yaskevich <xe...@apache.org>
Committed: Tue Aug 6 15:20:48 2013 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
build.xml | 4 ++--
lib/thrift-server-0.2.1.jar | Bin 123693 -> 0 bytes
lib/thrift-server-0.3.0.jar | Bin 0 -> 123898 bytes
.../cassandra/thrift/THsHaDisruptorServer.java | 3 ++-
5 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/19767fcd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index af94bc8..86401d9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,6 @@
2.0.0
* fix CAS contention timeout (CASSANDRA-5830)
+ * fix HsHa to respect max frame size (CASSANDRA-4573)
2.0.0-rc1
http://git-wip-us.apache.org/repos/asf/cassandra/blob/19767fcd/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index a40405e..a72adf4 100644
--- a/build.xml
+++ b/build.xml
@@ -353,7 +353,7 @@
<dependency groupId="com.googlecode.json-simple" artifactId="json-simple" version="1.1"/>
<dependency groupId="com.github.stephenc.high-scale-lib" artifactId="high-scale-lib" version="1.1.2"/>
<dependency groupId="com.github.stephenc" artifactId="jamm" version="0.2.5"/>
- <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server" version="0.2.1"/>
+ <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server" version="0.3.0"/>
<dependency groupId="org.yaml" artifactId="snakeyaml" version="1.6"/>
<dependency groupId="org.apache.thrift" artifactId="libthrift" version="0.9.0"/>
@@ -451,7 +451,7 @@
<dependency groupId="edu.stanford.ppl" artifactId="snaptree"/>
<dependency groupId="org.mindrot" artifactId="jbcrypt"/>
<dependency groupId="com.yammer.metrics" artifactId="metrics-core"/>
- <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server" version="0.2.1"/>
+ <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server" version="0.3.0"/>
<dependency groupId="log4j" artifactId="log4j"/>
<!-- cassandra has a hard dependency on log4j, so force slf4j's log4j provider at runtime -->
http://git-wip-us.apache.org/repos/asf/cassandra/blob/19767fcd/lib/thrift-server-0.2.1.jar
----------------------------------------------------------------------
diff --git a/lib/thrift-server-0.2.1.jar b/lib/thrift-server-0.2.1.jar
deleted file mode 100644
index 6b8de7d..0000000
Binary files a/lib/thrift-server-0.2.1.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/19767fcd/lib/thrift-server-0.3.0.jar
----------------------------------------------------------------------
diff --git a/lib/thrift-server-0.3.0.jar b/lib/thrift-server-0.3.0.jar
new file mode 100644
index 0000000..3c93bb9
Binary files /dev/null and b/lib/thrift-server-0.3.0.jar differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/19767fcd/src/java/org/apache/cassandra/thrift/THsHaDisruptorServer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/thrift/THsHaDisruptorServer.java b/src/java/org/apache/cassandra/thrift/THsHaDisruptorServer.java
index 57a7893..0df90ec 100644
--- a/src/java/org/apache/cassandra/thrift/THsHaDisruptorServer.java
+++ b/src/java/org/apache/cassandra/thrift/THsHaDisruptorServer.java
@@ -84,7 +84,8 @@ public class THsHaDisruptorServer extends TDisruptorServer
.outputTransportFactory(args.outTransportFactory)
.inputProtocolFactory(protocolFactory)
.outputProtocolFactory(protocolFactory)
- .processor(args.processor);
+ .processor(args.processor)
+ .maxFrameSizeInBytes(DatabaseDescriptor.getThriftFramedTransportSize());
return new THsHaDisruptorServer(serverArgs);
}
[05/12] git commit: add --migrate optionto sstableupgrade and
sstablescrub patch by Tyler Hobbs;
reviewed by Jeremiah Jordan for CASSANDRA-5831
Posted by jb...@apache.org.
add --migrate optionto sstableupgrade and sstablescrub
patch by Tyler Hobbs; reviewed by Jeremiah Jordan for CASSANDRA-5831
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7d45d2bd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7d45d2bd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7d45d2bd
Branch: refs/heads/cassandra-1.2
Commit: 7d45d2bde3724a15a9e0f68bb047a41853ccd340
Parents: 27943b1
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Aug 7 18:03:29 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Aug 7 18:03:29 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/SystemTable.java | 52 +++++++++++++++-----
.../cassandra/tools/StandaloneScrubber.java | 19 ++++++-
.../cassandra/tools/StandaloneUpgrader.java | 22 ++++++++-
4 files changed, 80 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 25c0217..859e7f9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
1.2.9
+ * add --migrate option to sstableupgrade and sstablescrub (CASSANDRA-5831)
* fix bulk-loading compressed sstables (CASSANDRA-5820)
* (Hadoop) fix quoting in CqlPagingRecordReader and CqlRecordWriter
(CASSANDRA-5824)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/db/SystemTable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/SystemTable.java b/src/java/org/apache/cassandra/db/SystemTable.java
index 4c74df5..318b34d 100644
--- a/src/java/org/apache/cassandra/db/SystemTable.java
+++ b/src/java/org/apache/cassandra/db/SystemTable.java
@@ -147,6 +147,7 @@ public class SystemTable
ColumnFamilyStore oldStatusCfs = table.getColumnFamilyStore(OLD_STATUS_CF);
if (oldStatusCfs.getSSTables().size() > 0)
{
+ logger.info("Old system data found in {}.{}; migrating to new format in {}.{}", Table.SYSTEM_KS, OLD_STATUS_CF, Table.SYSTEM_KS, LOCAL_CF);
SortedSet<ByteBuffer> cols = new TreeSet<ByteBuffer>(BytesType.instance);
cols.add(ByteBufferUtil.bytes("ClusterName"));
cols.add(ByteBufferUtil.bytes("Token"));
@@ -428,24 +429,51 @@ public class SystemTable
ex.initCause(err);
throw ex;
}
- ColumnFamilyStore cfs = table.getColumnFamilyStore(LOCAL_CF);
- String req = "SELECT cluster_name FROM system.%s WHERE key='%s'";
- UntypedResultSet result = processInternal(String.format(req, LOCAL_CF, LOCAL_KEY));
+ String savedClusterName;
- if (result.isEmpty() || !result.one().has("cluster_name"))
+ // See if there is still data in System.LocationInfo, indicating that the system data has not yet been
+ // upgraded by SystemTable.upgradeSystemData()
+ ColumnFamilyStore oldStatusCfs = table.getColumnFamilyStore(OLD_STATUS_CF);
+ if (oldStatusCfs.getSSTables().size() > 0)
+ {
+ logger.debug("Detected system data in {}.{}, checking saved cluster name", Table.SYSTEM_KS, OLD_STATUS_CF);
+ SortedSet<ByteBuffer> cols = new TreeSet<ByteBuffer>(BytesType.instance);
+ cols.add(ByteBufferUtil.bytes("ClusterName"));
+ QueryFilter filter = QueryFilter.getNamesFilter(decorate(ByteBufferUtil.bytes("L")), new QueryPath(OLD_STATUS_CF), cols);
+ ColumnFamily oldCf = oldStatusCfs.getColumnFamily(filter);
+ try
+ {
+ savedClusterName = ByteBufferUtil.string(oldCf.getColumn(ByteBufferUtil.bytes("ClusterName")).value());
+ }
+ catch (CharacterCodingException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ else
{
- // this is a brand new node
- if (!cfs.getSSTables().isEmpty())
- throw new ConfigurationException("Found system table files, but they couldn't be loaded!");
+ ColumnFamilyStore cfs = table.getColumnFamilyStore(LOCAL_CF);
- // no system files. this is a new node.
- req = "INSERT INTO system.%s (key, cluster_name) VALUES ('%s', '%s')";
- processInternal(String.format(req, LOCAL_CF, LOCAL_KEY, DatabaseDescriptor.getClusterName()));
- return;
+ String req = "SELECT cluster_name FROM system.%s WHERE key='%s'";
+ UntypedResultSet result = processInternal(String.format(req, LOCAL_CF, LOCAL_KEY));
+
+ if (result.isEmpty() || !result.one().has("cluster_name"))
+ {
+
+ // this is a brand new node
+ if (!cfs.getSSTables().isEmpty())
+ throw new ConfigurationException("Found system table files, but they couldn't be loaded!");
+
+ // no system files. this is a new node.
+ req = "INSERT INTO system.%s (key, cluster_name) VALUES ('%s', '%s')";
+ processInternal(String.format(req, LOCAL_CF, LOCAL_KEY, DatabaseDescriptor.getClusterName()));
+ return;
+ }
+
+ savedClusterName = result.one().getString("cluster_name");
}
- String savedClusterName = result.one().getString("cluster_name");
if (!DatabaseDescriptor.getClusterName().equals(savedClusterName))
throw new ConfigurationException("Saved cluster name " + savedClusterName + " != configured name " + DatabaseDescriptor.getClusterName());
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 0542756..d59a7d1 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -49,15 +49,30 @@ public class StandaloneScrubber
private static final String DEBUG_OPTION = "debug";
private static final String HELP_OPTION = "help";
private static final String MANIFEST_CHECK_OPTION = "manifest-check";
+ private static final String MIGRATE_OPTION = "migrate";
public static void main(String args[]) throws IOException
{
Options options = Options.parseArgs(args);
try
{
+ OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
+
// Migrate sstables from pre-#2749 to the correct location
if (Directories.sstablesNeedsMigration())
+ {
+ if (!options.migrate)
+ {
+ System.err.println("Detected a pre-1.1 data directory layout. For this tool to work, a migration " +
+ "must be performed to the 1.1+ format for directories and filenames. Re-run " +
+ TOOL_NAME + " with the --" + MIGRATE_OPTION + " option to automatically " +
+ "migrate *all* keyspaces and column families to the new layout.");
+ System.exit(1);
+ }
+ handler.output("Detected a pre-1.1 data directory layout. All keyspace and column family directories " +
+ "will be migrated to the 1.1+ format.");
Directories.migrateSSTables();
+ }
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
@@ -72,7 +87,6 @@ public class StandaloneScrubber
ColumnFamilyStore cfs = table.getColumnFamilyStore(options.cfName);
String snapshotName = "pre-scrub-" + System.currentTimeMillis();
- OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
@@ -184,6 +198,7 @@ public class StandaloneScrubber
public boolean debug;
public boolean verbose;
public boolean manifestCheckOnly;
+ public boolean migrate;
private Options(String tableName, String cfName)
{
@@ -222,6 +237,7 @@ public class StandaloneScrubber
opts.debug = cmd.hasOption(DEBUG_OPTION);
opts.verbose = cmd.hasOption(VERBOSE_OPTION);
opts.manifestCheckOnly = cmd.hasOption(MANIFEST_CHECK_OPTION);
+ opts.migrate = cmd.hasOption(MIGRATE_OPTION);
return opts;
}
@@ -246,6 +262,7 @@ public class StandaloneScrubber
options.addOption("v", VERBOSE_OPTION, "verbose output");
options.addOption("h", HELP_OPTION, "display this help message");
options.addOption("m", MANIFEST_CHECK_OPTION, "only check and repair the leveled manifest, without actually scrubbing the sstables");
+ options.addOption(null, MIGRATE_OPTION, "convert directory layout and filenames to 1.1+ structure");
return options;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
index 357e99c..16bf588 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
@@ -47,12 +47,30 @@ public class StandaloneUpgrader
private static final String TOOL_NAME = "sstableupgrade";
private static final String DEBUG_OPTION = "debug";
private static final String HELP_OPTION = "help";
+ private static final String MIGRATE_OPTION = "migrate";
public static void main(String args[]) throws IOException
{
Options options = Options.parseArgs(args);
try
{
+ OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
+ // Migrate sstables from pre-#2749 to the correct location
+ if (Directories.sstablesNeedsMigration())
+ {
+ if (!options.migrate)
+ {
+ System.err.println("Detected a pre-1.1 data directory layout. For this tool to work, a migration " +
+ "must be performed to the 1.1+ format for directories and filenames. Re-run " +
+ TOOL_NAME + " with the --" + MIGRATE_OPTION + " option to automatically " +
+ "migrate *all* keyspaces and column families to the new layout.");
+ System.exit(1);
+ }
+ handler.output("Detected a pre-1.1 data directory layout. All keyspace and column family directories " +
+ "will be migrated to the 1.1+ format.");
+ Directories.migrateSSTables();
+ }
+
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
@@ -64,7 +82,6 @@ public class StandaloneUpgrader
Table table = Table.openWithoutSSTables(options.keyspace);
ColumnFamilyStore cfs = table.getColumnFamilyStore(options.cf);
- OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
Directories.SSTableLister lister = cfs.directories.sstableLister();
if (options.snapshot != null)
lister.onlyBackups(true).snapshots(options.snapshot);
@@ -137,6 +154,7 @@ public class StandaloneUpgrader
public final String snapshot;
public boolean debug;
+ public boolean migrate;
private Options(String keyspace, String cf, String snapshot)
{
@@ -176,6 +194,7 @@ public class StandaloneUpgrader
Options opts = new Options(keyspace, cf, snapshot);
opts.debug = cmd.hasOption(DEBUG_OPTION);
+ opts.migrate = cmd.hasOption(MIGRATE_OPTION);
return opts;
}
@@ -197,6 +216,7 @@ public class StandaloneUpgrader
{
CmdLineOptions options = new CmdLineOptions();
options.addOption(null, DEBUG_OPTION, "display stack traces");
+ options.addOption(null, MIGRATE_OPTION, "convert directory layout and filenames to 1.1+ structure");
options.addOption("h", HELP_OPTION, "display this help message");
return options;
}
[10/12] git commit: Merge branch 'cassandra-2.0.0' into cassandra-2.0
Posted by jb...@apache.org.
Merge branch 'cassandra-2.0.0' into cassandra-2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/23becceb
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/23becceb
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/23becceb
Branch: refs/heads/trunk
Commit: 23becceb7b652a46fbf098882fbd01113a598e76
Parents: 18f0234 88fea9f
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Aug 7 18:05:10 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Aug 7 18:05:10 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
build.xml | 4 ++--
lib/thrift-server-0.2.1.jar | Bin 123693 -> 0 bytes
lib/thrift-server-0.3.0.jar | Bin 0 -> 123898 bytes
.../cassandra/thrift/THsHaDisruptorServer.java | 3 ++-
5 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/23becceb/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 4dca1b8,86401d9..30aa5e4
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,10 -1,6 +1,11 @@@
+2.0.1
+ * Notify indexer of columns shadowed by range tombstones (CASSANDRA-5614)
+ * Log Merkle tree stats (CASSANDRA-2698)
+
+
2.0.0
* fix CAS contention timeout (CASSANDRA-5830)
+ * fix HsHa to respect max frame size (CASSANDRA-4573)
2.0.0-rc1
[03/12] git commit: add --migrate optionto sstableupgrade and
sstablescrub patch by Tyler Hobbs;
reviewed by Jeremiah Jordan for CASSANDRA-5831
Posted by jb...@apache.org.
add --migrate optionto sstableupgrade and sstablescrub
patch by Tyler Hobbs; reviewed by Jeremiah Jordan for CASSANDRA-5831
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7d45d2bd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7d45d2bd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7d45d2bd
Branch: refs/heads/cassandra-2.0.0
Commit: 7d45d2bde3724a15a9e0f68bb047a41853ccd340
Parents: 27943b1
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Aug 7 18:03:29 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Aug 7 18:03:29 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/SystemTable.java | 52 +++++++++++++++-----
.../cassandra/tools/StandaloneScrubber.java | 19 ++++++-
.../cassandra/tools/StandaloneUpgrader.java | 22 ++++++++-
4 files changed, 80 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 25c0217..859e7f9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
1.2.9
+ * add --migrate option to sstableupgrade and sstablescrub (CASSANDRA-5831)
* fix bulk-loading compressed sstables (CASSANDRA-5820)
* (Hadoop) fix quoting in CqlPagingRecordReader and CqlRecordWriter
(CASSANDRA-5824)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/db/SystemTable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/SystemTable.java b/src/java/org/apache/cassandra/db/SystemTable.java
index 4c74df5..318b34d 100644
--- a/src/java/org/apache/cassandra/db/SystemTable.java
+++ b/src/java/org/apache/cassandra/db/SystemTable.java
@@ -147,6 +147,7 @@ public class SystemTable
ColumnFamilyStore oldStatusCfs = table.getColumnFamilyStore(OLD_STATUS_CF);
if (oldStatusCfs.getSSTables().size() > 0)
{
+ logger.info("Old system data found in {}.{}; migrating to new format in {}.{}", Table.SYSTEM_KS, OLD_STATUS_CF, Table.SYSTEM_KS, LOCAL_CF);
SortedSet<ByteBuffer> cols = new TreeSet<ByteBuffer>(BytesType.instance);
cols.add(ByteBufferUtil.bytes("ClusterName"));
cols.add(ByteBufferUtil.bytes("Token"));
@@ -428,24 +429,51 @@ public class SystemTable
ex.initCause(err);
throw ex;
}
- ColumnFamilyStore cfs = table.getColumnFamilyStore(LOCAL_CF);
- String req = "SELECT cluster_name FROM system.%s WHERE key='%s'";
- UntypedResultSet result = processInternal(String.format(req, LOCAL_CF, LOCAL_KEY));
+ String savedClusterName;
- if (result.isEmpty() || !result.one().has("cluster_name"))
+ // See if there is still data in System.LocationInfo, indicating that the system data has not yet been
+ // upgraded by SystemTable.upgradeSystemData()
+ ColumnFamilyStore oldStatusCfs = table.getColumnFamilyStore(OLD_STATUS_CF);
+ if (oldStatusCfs.getSSTables().size() > 0)
+ {
+ logger.debug("Detected system data in {}.{}, checking saved cluster name", Table.SYSTEM_KS, OLD_STATUS_CF);
+ SortedSet<ByteBuffer> cols = new TreeSet<ByteBuffer>(BytesType.instance);
+ cols.add(ByteBufferUtil.bytes("ClusterName"));
+ QueryFilter filter = QueryFilter.getNamesFilter(decorate(ByteBufferUtil.bytes("L")), new QueryPath(OLD_STATUS_CF), cols);
+ ColumnFamily oldCf = oldStatusCfs.getColumnFamily(filter);
+ try
+ {
+ savedClusterName = ByteBufferUtil.string(oldCf.getColumn(ByteBufferUtil.bytes("ClusterName")).value());
+ }
+ catch (CharacterCodingException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ else
{
- // this is a brand new node
- if (!cfs.getSSTables().isEmpty())
- throw new ConfigurationException("Found system table files, but they couldn't be loaded!");
+ ColumnFamilyStore cfs = table.getColumnFamilyStore(LOCAL_CF);
- // no system files. this is a new node.
- req = "INSERT INTO system.%s (key, cluster_name) VALUES ('%s', '%s')";
- processInternal(String.format(req, LOCAL_CF, LOCAL_KEY, DatabaseDescriptor.getClusterName()));
- return;
+ String req = "SELECT cluster_name FROM system.%s WHERE key='%s'";
+ UntypedResultSet result = processInternal(String.format(req, LOCAL_CF, LOCAL_KEY));
+
+ if (result.isEmpty() || !result.one().has("cluster_name"))
+ {
+
+ // this is a brand new node
+ if (!cfs.getSSTables().isEmpty())
+ throw new ConfigurationException("Found system table files, but they couldn't be loaded!");
+
+ // no system files. this is a new node.
+ req = "INSERT INTO system.%s (key, cluster_name) VALUES ('%s', '%s')";
+ processInternal(String.format(req, LOCAL_CF, LOCAL_KEY, DatabaseDescriptor.getClusterName()));
+ return;
+ }
+
+ savedClusterName = result.one().getString("cluster_name");
}
- String savedClusterName = result.one().getString("cluster_name");
if (!DatabaseDescriptor.getClusterName().equals(savedClusterName))
throw new ConfigurationException("Saved cluster name " + savedClusterName + " != configured name " + DatabaseDescriptor.getClusterName());
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 0542756..d59a7d1 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -49,15 +49,30 @@ public class StandaloneScrubber
private static final String DEBUG_OPTION = "debug";
private static final String HELP_OPTION = "help";
private static final String MANIFEST_CHECK_OPTION = "manifest-check";
+ private static final String MIGRATE_OPTION = "migrate";
public static void main(String args[]) throws IOException
{
Options options = Options.parseArgs(args);
try
{
+ OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
+
// Migrate sstables from pre-#2749 to the correct location
if (Directories.sstablesNeedsMigration())
+ {
+ if (!options.migrate)
+ {
+ System.err.println("Detected a pre-1.1 data directory layout. For this tool to work, a migration " +
+ "must be performed to the 1.1+ format for directories and filenames. Re-run " +
+ TOOL_NAME + " with the --" + MIGRATE_OPTION + " option to automatically " +
+ "migrate *all* keyspaces and column families to the new layout.");
+ System.exit(1);
+ }
+ handler.output("Detected a pre-1.1 data directory layout. All keyspace and column family directories " +
+ "will be migrated to the 1.1+ format.");
Directories.migrateSSTables();
+ }
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
@@ -72,7 +87,6 @@ public class StandaloneScrubber
ColumnFamilyStore cfs = table.getColumnFamilyStore(options.cfName);
String snapshotName = "pre-scrub-" + System.currentTimeMillis();
- OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
@@ -184,6 +198,7 @@ public class StandaloneScrubber
public boolean debug;
public boolean verbose;
public boolean manifestCheckOnly;
+ public boolean migrate;
private Options(String tableName, String cfName)
{
@@ -222,6 +237,7 @@ public class StandaloneScrubber
opts.debug = cmd.hasOption(DEBUG_OPTION);
opts.verbose = cmd.hasOption(VERBOSE_OPTION);
opts.manifestCheckOnly = cmd.hasOption(MANIFEST_CHECK_OPTION);
+ opts.migrate = cmd.hasOption(MIGRATE_OPTION);
return opts;
}
@@ -246,6 +262,7 @@ public class StandaloneScrubber
options.addOption("v", VERBOSE_OPTION, "verbose output");
options.addOption("h", HELP_OPTION, "display this help message");
options.addOption("m", MANIFEST_CHECK_OPTION, "only check and repair the leveled manifest, without actually scrubbing the sstables");
+ options.addOption(null, MIGRATE_OPTION, "convert directory layout and filenames to 1.1+ structure");
return options;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
index 357e99c..16bf588 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
@@ -47,12 +47,30 @@ public class StandaloneUpgrader
private static final String TOOL_NAME = "sstableupgrade";
private static final String DEBUG_OPTION = "debug";
private static final String HELP_OPTION = "help";
+ private static final String MIGRATE_OPTION = "migrate";
public static void main(String args[]) throws IOException
{
Options options = Options.parseArgs(args);
try
{
+ OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
+ // Migrate sstables from pre-#2749 to the correct location
+ if (Directories.sstablesNeedsMigration())
+ {
+ if (!options.migrate)
+ {
+ System.err.println("Detected a pre-1.1 data directory layout. For this tool to work, a migration " +
+ "must be performed to the 1.1+ format for directories and filenames. Re-run " +
+ TOOL_NAME + " with the --" + MIGRATE_OPTION + " option to automatically " +
+ "migrate *all* keyspaces and column families to the new layout.");
+ System.exit(1);
+ }
+ handler.output("Detected a pre-1.1 data directory layout. All keyspace and column family directories " +
+ "will be migrated to the 1.1+ format.");
+ Directories.migrateSSTables();
+ }
+
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
@@ -64,7 +82,6 @@ public class StandaloneUpgrader
Table table = Table.openWithoutSSTables(options.keyspace);
ColumnFamilyStore cfs = table.getColumnFamilyStore(options.cf);
- OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
Directories.SSTableLister lister = cfs.directories.sstableLister();
if (options.snapshot != null)
lister.onlyBackups(true).snapshots(options.snapshot);
@@ -137,6 +154,7 @@ public class StandaloneUpgrader
public final String snapshot;
public boolean debug;
+ public boolean migrate;
private Options(String keyspace, String cf, String snapshot)
{
@@ -176,6 +194,7 @@ public class StandaloneUpgrader
Options opts = new Options(keyspace, cf, snapshot);
opts.debug = cmd.hasOption(DEBUG_OPTION);
+ opts.migrate = cmd.hasOption(MIGRATE_OPTION);
return opts;
}
@@ -197,6 +216,7 @@ public class StandaloneUpgrader
{
CmdLineOptions options = new CmdLineOptions();
options.addOption(null, DEBUG_OPTION, "display stack traces");
+ options.addOption(null, MIGRATE_OPTION, "convert directory layout and filenames to 1.1+ structure");
options.addOption("h", HELP_OPTION, "display this help message");
return options;
}
[06/12] git commit: add --migrate optionto sstableupgrade and
sstablescrub patch by Tyler Hobbs;
reviewed by Jeremiah Jordan for CASSANDRA-5831
Posted by jb...@apache.org.
add --migrate optionto sstableupgrade and sstablescrub
patch by Tyler Hobbs; reviewed by Jeremiah Jordan for CASSANDRA-5831
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7d45d2bd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7d45d2bd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7d45d2bd
Branch: refs/heads/cassandra-2.0
Commit: 7d45d2bde3724a15a9e0f68bb047a41853ccd340
Parents: 27943b1
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Aug 7 18:03:29 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Aug 7 18:03:29 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/SystemTable.java | 52 +++++++++++++++-----
.../cassandra/tools/StandaloneScrubber.java | 19 ++++++-
.../cassandra/tools/StandaloneUpgrader.java | 22 ++++++++-
4 files changed, 80 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 25c0217..859e7f9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
1.2.9
+ * add --migrate option to sstableupgrade and sstablescrub (CASSANDRA-5831)
* fix bulk-loading compressed sstables (CASSANDRA-5820)
* (Hadoop) fix quoting in CqlPagingRecordReader and CqlRecordWriter
(CASSANDRA-5824)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/db/SystemTable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/SystemTable.java b/src/java/org/apache/cassandra/db/SystemTable.java
index 4c74df5..318b34d 100644
--- a/src/java/org/apache/cassandra/db/SystemTable.java
+++ b/src/java/org/apache/cassandra/db/SystemTable.java
@@ -147,6 +147,7 @@ public class SystemTable
ColumnFamilyStore oldStatusCfs = table.getColumnFamilyStore(OLD_STATUS_CF);
if (oldStatusCfs.getSSTables().size() > 0)
{
+ logger.info("Old system data found in {}.{}; migrating to new format in {}.{}", Table.SYSTEM_KS, OLD_STATUS_CF, Table.SYSTEM_KS, LOCAL_CF);
SortedSet<ByteBuffer> cols = new TreeSet<ByteBuffer>(BytesType.instance);
cols.add(ByteBufferUtil.bytes("ClusterName"));
cols.add(ByteBufferUtil.bytes("Token"));
@@ -428,24 +429,51 @@ public class SystemTable
ex.initCause(err);
throw ex;
}
- ColumnFamilyStore cfs = table.getColumnFamilyStore(LOCAL_CF);
- String req = "SELECT cluster_name FROM system.%s WHERE key='%s'";
- UntypedResultSet result = processInternal(String.format(req, LOCAL_CF, LOCAL_KEY));
+ String savedClusterName;
- if (result.isEmpty() || !result.one().has("cluster_name"))
+ // See if there is still data in System.LocationInfo, indicating that the system data has not yet been
+ // upgraded by SystemTable.upgradeSystemData()
+ ColumnFamilyStore oldStatusCfs = table.getColumnFamilyStore(OLD_STATUS_CF);
+ if (oldStatusCfs.getSSTables().size() > 0)
+ {
+ logger.debug("Detected system data in {}.{}, checking saved cluster name", Table.SYSTEM_KS, OLD_STATUS_CF);
+ SortedSet<ByteBuffer> cols = new TreeSet<ByteBuffer>(BytesType.instance);
+ cols.add(ByteBufferUtil.bytes("ClusterName"));
+ QueryFilter filter = QueryFilter.getNamesFilter(decorate(ByteBufferUtil.bytes("L")), new QueryPath(OLD_STATUS_CF), cols);
+ ColumnFamily oldCf = oldStatusCfs.getColumnFamily(filter);
+ try
+ {
+ savedClusterName = ByteBufferUtil.string(oldCf.getColumn(ByteBufferUtil.bytes("ClusterName")).value());
+ }
+ catch (CharacterCodingException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ else
{
- // this is a brand new node
- if (!cfs.getSSTables().isEmpty())
- throw new ConfigurationException("Found system table files, but they couldn't be loaded!");
+ ColumnFamilyStore cfs = table.getColumnFamilyStore(LOCAL_CF);
- // no system files. this is a new node.
- req = "INSERT INTO system.%s (key, cluster_name) VALUES ('%s', '%s')";
- processInternal(String.format(req, LOCAL_CF, LOCAL_KEY, DatabaseDescriptor.getClusterName()));
- return;
+ String req = "SELECT cluster_name FROM system.%s WHERE key='%s'";
+ UntypedResultSet result = processInternal(String.format(req, LOCAL_CF, LOCAL_KEY));
+
+ if (result.isEmpty() || !result.one().has("cluster_name"))
+ {
+
+ // this is a brand new node
+ if (!cfs.getSSTables().isEmpty())
+ throw new ConfigurationException("Found system table files, but they couldn't be loaded!");
+
+ // no system files. this is a new node.
+ req = "INSERT INTO system.%s (key, cluster_name) VALUES ('%s', '%s')";
+ processInternal(String.format(req, LOCAL_CF, LOCAL_KEY, DatabaseDescriptor.getClusterName()));
+ return;
+ }
+
+ savedClusterName = result.one().getString("cluster_name");
}
- String savedClusterName = result.one().getString("cluster_name");
if (!DatabaseDescriptor.getClusterName().equals(savedClusterName))
throw new ConfigurationException("Saved cluster name " + savedClusterName + " != configured name " + DatabaseDescriptor.getClusterName());
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 0542756..d59a7d1 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -49,15 +49,30 @@ public class StandaloneScrubber
private static final String DEBUG_OPTION = "debug";
private static final String HELP_OPTION = "help";
private static final String MANIFEST_CHECK_OPTION = "manifest-check";
+ private static final String MIGRATE_OPTION = "migrate";
public static void main(String args[]) throws IOException
{
Options options = Options.parseArgs(args);
try
{
+ OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
+
// Migrate sstables from pre-#2749 to the correct location
if (Directories.sstablesNeedsMigration())
+ {
+ if (!options.migrate)
+ {
+ System.err.println("Detected a pre-1.1 data directory layout. For this tool to work, a migration " +
+ "must be performed to the 1.1+ format for directories and filenames. Re-run " +
+ TOOL_NAME + " with the --" + MIGRATE_OPTION + " option to automatically " +
+ "migrate *all* keyspaces and column families to the new layout.");
+ System.exit(1);
+ }
+ handler.output("Detected a pre-1.1 data directory layout. All keyspace and column family directories " +
+ "will be migrated to the 1.1+ format.");
Directories.migrateSSTables();
+ }
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
@@ -72,7 +87,6 @@ public class StandaloneScrubber
ColumnFamilyStore cfs = table.getColumnFamilyStore(options.cfName);
String snapshotName = "pre-scrub-" + System.currentTimeMillis();
- OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
@@ -184,6 +198,7 @@ public class StandaloneScrubber
public boolean debug;
public boolean verbose;
public boolean manifestCheckOnly;
+ public boolean migrate;
private Options(String tableName, String cfName)
{
@@ -222,6 +237,7 @@ public class StandaloneScrubber
opts.debug = cmd.hasOption(DEBUG_OPTION);
opts.verbose = cmd.hasOption(VERBOSE_OPTION);
opts.manifestCheckOnly = cmd.hasOption(MANIFEST_CHECK_OPTION);
+ opts.migrate = cmd.hasOption(MIGRATE_OPTION);
return opts;
}
@@ -246,6 +262,7 @@ public class StandaloneScrubber
options.addOption("v", VERBOSE_OPTION, "verbose output");
options.addOption("h", HELP_OPTION, "display this help message");
options.addOption("m", MANIFEST_CHECK_OPTION, "only check and repair the leveled manifest, without actually scrubbing the sstables");
+ options.addOption(null, MIGRATE_OPTION, "convert directory layout and filenames to 1.1+ structure");
return options;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
index 357e99c..16bf588 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
@@ -47,12 +47,30 @@ public class StandaloneUpgrader
private static final String TOOL_NAME = "sstableupgrade";
private static final String DEBUG_OPTION = "debug";
private static final String HELP_OPTION = "help";
+ private static final String MIGRATE_OPTION = "migrate";
public static void main(String args[]) throws IOException
{
Options options = Options.parseArgs(args);
try
{
+ OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
+ // Migrate sstables from pre-#2749 to the correct location
+ if (Directories.sstablesNeedsMigration())
+ {
+ if (!options.migrate)
+ {
+ System.err.println("Detected a pre-1.1 data directory layout. For this tool to work, a migration " +
+ "must be performed to the 1.1+ format for directories and filenames. Re-run " +
+ TOOL_NAME + " with the --" + MIGRATE_OPTION + " option to automatically " +
+ "migrate *all* keyspaces and column families to the new layout.");
+ System.exit(1);
+ }
+ handler.output("Detected a pre-1.1 data directory layout. All keyspace and column family directories " +
+ "will be migrated to the 1.1+ format.");
+ Directories.migrateSSTables();
+ }
+
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
@@ -64,7 +82,6 @@ public class StandaloneUpgrader
Table table = Table.openWithoutSSTables(options.keyspace);
ColumnFamilyStore cfs = table.getColumnFamilyStore(options.cf);
- OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
Directories.SSTableLister lister = cfs.directories.sstableLister();
if (options.snapshot != null)
lister.onlyBackups(true).snapshots(options.snapshot);
@@ -137,6 +154,7 @@ public class StandaloneUpgrader
public final String snapshot;
public boolean debug;
+ public boolean migrate;
private Options(String keyspace, String cf, String snapshot)
{
@@ -176,6 +194,7 @@ public class StandaloneUpgrader
Options opts = new Options(keyspace, cf, snapshot);
opts.debug = cmd.hasOption(DEBUG_OPTION);
+ opts.migrate = cmd.hasOption(MIGRATE_OPTION);
return opts;
}
@@ -197,6 +216,7 @@ public class StandaloneUpgrader
{
CmdLineOptions options = new CmdLineOptions();
options.addOption(null, DEBUG_OPTION, "display stack traces");
+ options.addOption(null, MIGRATE_OPTION, "convert directory layout and filenames to 1.1+ structure");
options.addOption("h", HELP_OPTION, "display this help message");
return options;
}
[11/12] git commit: Merge branch 'cassandra-2.0.0' into cassandra-2.0
Posted by jb...@apache.org.
Merge branch 'cassandra-2.0.0' into cassandra-2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/23becceb
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/23becceb
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/23becceb
Branch: refs/heads/cassandra-2.0
Commit: 23becceb7b652a46fbf098882fbd01113a598e76
Parents: 18f0234 88fea9f
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Aug 7 18:05:10 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Aug 7 18:05:10 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
build.xml | 4 ++--
lib/thrift-server-0.2.1.jar | Bin 123693 -> 0 bytes
lib/thrift-server-0.3.0.jar | Bin 0 -> 123898 bytes
.../cassandra/thrift/THsHaDisruptorServer.java | 3 ++-
5 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/23becceb/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 4dca1b8,86401d9..30aa5e4
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,10 -1,6 +1,11 @@@
+2.0.1
+ * Notify indexer of columns shadowed by range tombstones (CASSANDRA-5614)
+ * Log Merkle tree stats (CASSANDRA-2698)
+
+
2.0.0
* fix CAS contention timeout (CASSANDRA-5830)
+ * fix HsHa to respect max frame size (CASSANDRA-4573)
2.0.0-rc1
[09/12] git commit: Merge branch 'cassandra-1.2' into cassandra-2.0.0
Posted by jb...@apache.org.
Merge branch 'cassandra-1.2' into cassandra-2.0.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/88fea9fe
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/88fea9fe
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/88fea9fe
Branch: refs/heads/trunk
Commit: 88fea9fe736cae5920a41adc631e734384df0519
Parents: 19767fc 7d45d2b
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Aug 7 18:04:55 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Aug 7 18:04:55 2013 -0500
----------------------------------------------------------------------
----------------------------------------------------------------------
[04/12] git commit: add --migrate optionto sstableupgrade and
sstablescrub patch by Tyler Hobbs;
reviewed by Jeremiah Jordan for CASSANDRA-5831
Posted by jb...@apache.org.
add --migrate optionto sstableupgrade and sstablescrub
patch by Tyler Hobbs; reviewed by Jeremiah Jordan for CASSANDRA-5831
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7d45d2bd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7d45d2bd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7d45d2bd
Branch: refs/heads/trunk
Commit: 7d45d2bde3724a15a9e0f68bb047a41853ccd340
Parents: 27943b1
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Aug 7 18:03:29 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Aug 7 18:03:29 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/cassandra/db/SystemTable.java | 52 +++++++++++++++-----
.../cassandra/tools/StandaloneScrubber.java | 19 ++++++-
.../cassandra/tools/StandaloneUpgrader.java | 22 ++++++++-
4 files changed, 80 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 25c0217..859e7f9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
1.2.9
+ * add --migrate option to sstableupgrade and sstablescrub (CASSANDRA-5831)
* fix bulk-loading compressed sstables (CASSANDRA-5820)
* (Hadoop) fix quoting in CqlPagingRecordReader and CqlRecordWriter
(CASSANDRA-5824)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/db/SystemTable.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/SystemTable.java b/src/java/org/apache/cassandra/db/SystemTable.java
index 4c74df5..318b34d 100644
--- a/src/java/org/apache/cassandra/db/SystemTable.java
+++ b/src/java/org/apache/cassandra/db/SystemTable.java
@@ -147,6 +147,7 @@ public class SystemTable
ColumnFamilyStore oldStatusCfs = table.getColumnFamilyStore(OLD_STATUS_CF);
if (oldStatusCfs.getSSTables().size() > 0)
{
+ logger.info("Old system data found in {}.{}; migrating to new format in {}.{}", Table.SYSTEM_KS, OLD_STATUS_CF, Table.SYSTEM_KS, LOCAL_CF);
SortedSet<ByteBuffer> cols = new TreeSet<ByteBuffer>(BytesType.instance);
cols.add(ByteBufferUtil.bytes("ClusterName"));
cols.add(ByteBufferUtil.bytes("Token"));
@@ -428,24 +429,51 @@ public class SystemTable
ex.initCause(err);
throw ex;
}
- ColumnFamilyStore cfs = table.getColumnFamilyStore(LOCAL_CF);
- String req = "SELECT cluster_name FROM system.%s WHERE key='%s'";
- UntypedResultSet result = processInternal(String.format(req, LOCAL_CF, LOCAL_KEY));
+ String savedClusterName;
- if (result.isEmpty() || !result.one().has("cluster_name"))
+ // See if there is still data in System.LocationInfo, indicating that the system data has not yet been
+ // upgraded by SystemTable.upgradeSystemData()
+ ColumnFamilyStore oldStatusCfs = table.getColumnFamilyStore(OLD_STATUS_CF);
+ if (oldStatusCfs.getSSTables().size() > 0)
+ {
+ logger.debug("Detected system data in {}.{}, checking saved cluster name", Table.SYSTEM_KS, OLD_STATUS_CF);
+ SortedSet<ByteBuffer> cols = new TreeSet<ByteBuffer>(BytesType.instance);
+ cols.add(ByteBufferUtil.bytes("ClusterName"));
+ QueryFilter filter = QueryFilter.getNamesFilter(decorate(ByteBufferUtil.bytes("L")), new QueryPath(OLD_STATUS_CF), cols);
+ ColumnFamily oldCf = oldStatusCfs.getColumnFamily(filter);
+ try
+ {
+ savedClusterName = ByteBufferUtil.string(oldCf.getColumn(ByteBufferUtil.bytes("ClusterName")).value());
+ }
+ catch (CharacterCodingException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+ else
{
- // this is a brand new node
- if (!cfs.getSSTables().isEmpty())
- throw new ConfigurationException("Found system table files, but they couldn't be loaded!");
+ ColumnFamilyStore cfs = table.getColumnFamilyStore(LOCAL_CF);
- // no system files. this is a new node.
- req = "INSERT INTO system.%s (key, cluster_name) VALUES ('%s', '%s')";
- processInternal(String.format(req, LOCAL_CF, LOCAL_KEY, DatabaseDescriptor.getClusterName()));
- return;
+ String req = "SELECT cluster_name FROM system.%s WHERE key='%s'";
+ UntypedResultSet result = processInternal(String.format(req, LOCAL_CF, LOCAL_KEY));
+
+ if (result.isEmpty() || !result.one().has("cluster_name"))
+ {
+
+ // this is a brand new node
+ if (!cfs.getSSTables().isEmpty())
+ throw new ConfigurationException("Found system table files, but they couldn't be loaded!");
+
+ // no system files. this is a new node.
+ req = "INSERT INTO system.%s (key, cluster_name) VALUES ('%s', '%s')";
+ processInternal(String.format(req, LOCAL_CF, LOCAL_KEY, DatabaseDescriptor.getClusterName()));
+ return;
+ }
+
+ savedClusterName = result.one().getString("cluster_name");
}
- String savedClusterName = result.one().getString("cluster_name");
if (!DatabaseDescriptor.getClusterName().equals(savedClusterName))
throw new ConfigurationException("Saved cluster name " + savedClusterName + " != configured name " + DatabaseDescriptor.getClusterName());
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 0542756..d59a7d1 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -49,15 +49,30 @@ public class StandaloneScrubber
private static final String DEBUG_OPTION = "debug";
private static final String HELP_OPTION = "help";
private static final String MANIFEST_CHECK_OPTION = "manifest-check";
+ private static final String MIGRATE_OPTION = "migrate";
public static void main(String args[]) throws IOException
{
Options options = Options.parseArgs(args);
try
{
+ OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
+
// Migrate sstables from pre-#2749 to the correct location
if (Directories.sstablesNeedsMigration())
+ {
+ if (!options.migrate)
+ {
+ System.err.println("Detected a pre-1.1 data directory layout. For this tool to work, a migration " +
+ "must be performed to the 1.1+ format for directories and filenames. Re-run " +
+ TOOL_NAME + " with the --" + MIGRATE_OPTION + " option to automatically " +
+ "migrate *all* keyspaces and column families to the new layout.");
+ System.exit(1);
+ }
+ handler.output("Detected a pre-1.1 data directory layout. All keyspace and column family directories " +
+ "will be migrated to the 1.1+ format.");
Directories.migrateSSTables();
+ }
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
@@ -72,7 +87,6 @@ public class StandaloneScrubber
ColumnFamilyStore cfs = table.getColumnFamilyStore(options.cfName);
String snapshotName = "pre-scrub-" + System.currentTimeMillis();
- OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
@@ -184,6 +198,7 @@ public class StandaloneScrubber
public boolean debug;
public boolean verbose;
public boolean manifestCheckOnly;
+ public boolean migrate;
private Options(String tableName, String cfName)
{
@@ -222,6 +237,7 @@ public class StandaloneScrubber
opts.debug = cmd.hasOption(DEBUG_OPTION);
opts.verbose = cmd.hasOption(VERBOSE_OPTION);
opts.manifestCheckOnly = cmd.hasOption(MANIFEST_CHECK_OPTION);
+ opts.migrate = cmd.hasOption(MIGRATE_OPTION);
return opts;
}
@@ -246,6 +262,7 @@ public class StandaloneScrubber
options.addOption("v", VERBOSE_OPTION, "verbose output");
options.addOption("h", HELP_OPTION, "display this help message");
options.addOption("m", MANIFEST_CHECK_OPTION, "only check and repair the leveled manifest, without actually scrubbing the sstables");
+ options.addOption(null, MIGRATE_OPTION, "convert directory layout and filenames to 1.1+ structure");
return options;
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/7d45d2bd/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
index 357e99c..16bf588 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
@@ -47,12 +47,30 @@ public class StandaloneUpgrader
private static final String TOOL_NAME = "sstableupgrade";
private static final String DEBUG_OPTION = "debug";
private static final String HELP_OPTION = "help";
+ private static final String MIGRATE_OPTION = "migrate";
public static void main(String args[]) throws IOException
{
Options options = Options.parseArgs(args);
try
{
+ OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
+ // Migrate sstables from pre-#2749 to the correct location
+ if (Directories.sstablesNeedsMigration())
+ {
+ if (!options.migrate)
+ {
+ System.err.println("Detected a pre-1.1 data directory layout. For this tool to work, a migration " +
+ "must be performed to the 1.1+ format for directories and filenames. Re-run " +
+ TOOL_NAME + " with the --" + MIGRATE_OPTION + " option to automatically " +
+ "migrate *all* keyspaces and column families to the new layout.");
+ System.exit(1);
+ }
+ handler.output("Detected a pre-1.1 data directory layout. All keyspace and column family directories " +
+ "will be migrated to the 1.1+ format.");
+ Directories.migrateSSTables();
+ }
+
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
@@ -64,7 +82,6 @@ public class StandaloneUpgrader
Table table = Table.openWithoutSSTables(options.keyspace);
ColumnFamilyStore cfs = table.getColumnFamilyStore(options.cf);
- OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
Directories.SSTableLister lister = cfs.directories.sstableLister();
if (options.snapshot != null)
lister.onlyBackups(true).snapshots(options.snapshot);
@@ -137,6 +154,7 @@ public class StandaloneUpgrader
public final String snapshot;
public boolean debug;
+ public boolean migrate;
private Options(String keyspace, String cf, String snapshot)
{
@@ -176,6 +194,7 @@ public class StandaloneUpgrader
Options opts = new Options(keyspace, cf, snapshot);
opts.debug = cmd.hasOption(DEBUG_OPTION);
+ opts.migrate = cmd.hasOption(MIGRATE_OPTION);
return opts;
}
@@ -197,6 +216,7 @@ public class StandaloneUpgrader
{
CmdLineOptions options = new CmdLineOptions();
options.addOption(null, DEBUG_OPTION, "display stack traces");
+ options.addOption(null, MIGRATE_OPTION, "convert directory layout and filenames to 1.1+ structure");
options.addOption("h", HELP_OPTION, "display this help message");
return options;
}
[02/12] git commit: fix HsHa to respect max frame size patch by Pavel
Yaskevich; reviewed by Tyler Hobbs (CASSANDRA-4573)
Posted by jb...@apache.org.
fix HsHa to respect max frame size
patch by Pavel Yaskevich; reviewed by Tyler Hobbs (CASSANDRA-4573)
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/19767fcd
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/19767fcd
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/19767fcd
Branch: refs/heads/trunk
Commit: 19767fcd173418002e82a10e1b08a2231bb29e99
Parents: af85f23
Author: Pavel Yaskevich <xe...@apache.org>
Authored: Tue Aug 6 15:20:48 2013 -0700
Committer: Pavel Yaskevich <xe...@apache.org>
Committed: Tue Aug 6 15:20:48 2013 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
build.xml | 4 ++--
lib/thrift-server-0.2.1.jar | Bin 123693 -> 0 bytes
lib/thrift-server-0.3.0.jar | Bin 0 -> 123898 bytes
.../cassandra/thrift/THsHaDisruptorServer.java | 3 ++-
5 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/19767fcd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index af94bc8..86401d9 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,6 @@
2.0.0
* fix CAS contention timeout (CASSANDRA-5830)
+ * fix HsHa to respect max frame size (CASSANDRA-4573)
2.0.0-rc1
http://git-wip-us.apache.org/repos/asf/cassandra/blob/19767fcd/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index a40405e..a72adf4 100644
--- a/build.xml
+++ b/build.xml
@@ -353,7 +353,7 @@
<dependency groupId="com.googlecode.json-simple" artifactId="json-simple" version="1.1"/>
<dependency groupId="com.github.stephenc.high-scale-lib" artifactId="high-scale-lib" version="1.1.2"/>
<dependency groupId="com.github.stephenc" artifactId="jamm" version="0.2.5"/>
- <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server" version="0.2.1"/>
+ <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server" version="0.3.0"/>
<dependency groupId="org.yaml" artifactId="snakeyaml" version="1.6"/>
<dependency groupId="org.apache.thrift" artifactId="libthrift" version="0.9.0"/>
@@ -451,7 +451,7 @@
<dependency groupId="edu.stanford.ppl" artifactId="snaptree"/>
<dependency groupId="org.mindrot" artifactId="jbcrypt"/>
<dependency groupId="com.yammer.metrics" artifactId="metrics-core"/>
- <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server" version="0.2.1"/>
+ <dependency groupId="com.thinkaurelius.thrift" artifactId="thrift-server" version="0.3.0"/>
<dependency groupId="log4j" artifactId="log4j"/>
<!-- cassandra has a hard dependency on log4j, so force slf4j's log4j provider at runtime -->
http://git-wip-us.apache.org/repos/asf/cassandra/blob/19767fcd/lib/thrift-server-0.2.1.jar
----------------------------------------------------------------------
diff --git a/lib/thrift-server-0.2.1.jar b/lib/thrift-server-0.2.1.jar
deleted file mode 100644
index 6b8de7d..0000000
Binary files a/lib/thrift-server-0.2.1.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/19767fcd/lib/thrift-server-0.3.0.jar
----------------------------------------------------------------------
diff --git a/lib/thrift-server-0.3.0.jar b/lib/thrift-server-0.3.0.jar
new file mode 100644
index 0000000..3c93bb9
Binary files /dev/null and b/lib/thrift-server-0.3.0.jar differ
http://git-wip-us.apache.org/repos/asf/cassandra/blob/19767fcd/src/java/org/apache/cassandra/thrift/THsHaDisruptorServer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/thrift/THsHaDisruptorServer.java b/src/java/org/apache/cassandra/thrift/THsHaDisruptorServer.java
index 57a7893..0df90ec 100644
--- a/src/java/org/apache/cassandra/thrift/THsHaDisruptorServer.java
+++ b/src/java/org/apache/cassandra/thrift/THsHaDisruptorServer.java
@@ -84,7 +84,8 @@ public class THsHaDisruptorServer extends TDisruptorServer
.outputTransportFactory(args.outTransportFactory)
.inputProtocolFactory(protocolFactory)
.outputProtocolFactory(protocolFactory)
- .processor(args.processor);
+ .processor(args.processor)
+ .maxFrameSizeInBytes(DatabaseDescriptor.getThriftFramedTransportSize());
return new THsHaDisruptorServer(serverArgs);
}
[08/12] git commit: Merge branch 'cassandra-1.2' into cassandra-2.0.0
Posted by jb...@apache.org.
Merge branch 'cassandra-1.2' into cassandra-2.0.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/88fea9fe
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/88fea9fe
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/88fea9fe
Branch: refs/heads/cassandra-2.0.0
Commit: 88fea9fe736cae5920a41adc631e734384df0519
Parents: 19767fc 7d45d2b
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Aug 7 18:04:55 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Aug 7 18:04:55 2013 -0500
----------------------------------------------------------------------
----------------------------------------------------------------------
[07/12] git commit: Merge branch 'cassandra-1.2' into cassandra-2.0.0
Posted by jb...@apache.org.
Merge branch 'cassandra-1.2' into cassandra-2.0.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/88fea9fe
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/88fea9fe
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/88fea9fe
Branch: refs/heads/cassandra-2.0
Commit: 88fea9fe736cae5920a41adc631e734384df0519
Parents: 19767fc 7d45d2b
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Aug 7 18:04:55 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Aug 7 18:04:55 2013 -0500
----------------------------------------------------------------------
----------------------------------------------------------------------
[12/12] git commit: Merge branch 'cassandra-2.0' into trunk
Posted by jb...@apache.org.
Merge branch 'cassandra-2.0' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/66182082
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/66182082
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/66182082
Branch: refs/heads/trunk
Commit: 66182082cf6b02d1d3a50c48e99e52a4ccd795c9
Parents: 9cf1068 23becce
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Aug 7 18:05:23 2013 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Aug 7 18:05:23 2013 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
build.xml | 4 ++--
lib/thrift-server-0.2.1.jar | Bin 123693 -> 0 bytes
lib/thrift-server-0.3.0.jar | Bin 0 -> 123898 bytes
.../cassandra/thrift/THsHaDisruptorServer.java | 3 ++-
5 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------