You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2015/03/02 12:51:26 UTC
[2/2] cassandra git commit: Merge branch 'cassandra-2.0' into
cassandra-2.1
Merge branch 'cassandra-2.0' into cassandra-2.1
Conflicts:
CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/1e74dd0d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/1e74dd0d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/1e74dd0d
Branch: refs/heads/cassandra-2.1
Commit: 1e74dd0d10320f2155e31a152bd669cf9b6140f1
Parents: e76ebce 33a3a09
Author: Marcus Eriksson <ma...@apache.org>
Authored: Mon Mar 2 11:49:11 2015 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Mar 2 12:49:12 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/tools/SSTableOfflineRelevel.java | 193 +++++++++++++++++++
tools/bin/sstableofflinerelevel | 54 ++++++
3 files changed, 248 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1e74dd0d/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index a3e52f7,f98bb3f..15a5a61
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,24 -1,5 +1,25 @@@
-2.0.13:
+2.1.4
+ * Fix commitlog getPendingTasks to not increment (CASSANDRA-8856)
+ * Fix parallelism adjustment in range and secondary index queries
+ when the first fetch does not satisfy the limit (CASSANDRA-8856)
+ * Check if the filtered sstables is non-empty in STCS (CASSANDRA-8843)
+ * Upgrade java-driver used for cassandra-stress (CASSANDRA-8842)
+ * Fix CommitLog.forceRecycleAllSegments() memory access error (CASSANDRA-8812)
+ * Improve assertions in Memory (CASSANDRA-8792)
+ * Fix SSTableRewriter cleanup (CASSANDRA-8802)
+ * Introduce SafeMemory for CompressionMetadata.Writer (CASSANDRA-8758)
+ * 'nodetool info' prints exception against older node (CASSANDRA-8796)
+ * Ensure SSTableReader.last corresponds exactly with the file end (CASSANDRA-8750)
+ * Make SSTableWriter.openEarly more robust and obvious (CASSANDRA-8747)
+ * Enforce SSTableReader.first/last (CASSANDRA-8744)
+ * Cleanup SegmentedFile API (CASSANDRA-8749)
+ * Avoid overlap with early compaction replacement (CASSANDRA-8683)
+ * Safer Resource Management++ (CASSANDRA-8707)
+ * Write partition size estimates into a system table (CASSANDRA-7688)
+ * cqlsh: Fix keys() and full() collection indexes in DESCRIBE output
+ (CASSANDRA-8154)
+Merged from 2.0:
+ * Add offline tool to relevel sstables (CASSANDRA-8301)
* Preserve stream ID for more protocol errors (CASSANDRA-8848)
* Fix combining token() function with multi-column relations on
clustering columns (CASSANDRA-8797)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1e74dd0d/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
index 0000000,3fb2f7a..4c863c6
mode 000000,100644..100644
--- a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
+++ b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
@@@ -1,0 -1,191 +1,193 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.cassandra.tools;
+
+ import java.io.IOException;
+ import java.io.PrintStream;
+ import java.util.ArrayList;
+ import java.util.Collections;
+ import java.util.Comparator;
+ import java.util.HashSet;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Set;
+
+ import org.apache.cassandra.config.DatabaseDescriptor;
+ import org.apache.cassandra.config.Schema;
++import org.apache.cassandra.db.ColumnFamilyStore;
+ import org.apache.cassandra.db.DecoratedKey;
+ import org.apache.cassandra.db.Directories;
+ import org.apache.cassandra.db.Keyspace;
+ import org.apache.cassandra.db.compaction.LeveledManifest;
+ import org.apache.cassandra.io.sstable.Component;
+ import org.apache.cassandra.io.sstable.Descriptor;
+ import org.apache.cassandra.io.sstable.SSTableReader;
+ import org.apache.cassandra.utils.Pair;
+
+ /**
+ * Create a decent leveling for the given keyspace/column family
+ *
+ * Approach is to sort the sstables by their last token
+ *
+ * given an original leveling like this (note that [ ] indicates token boundaries, not sstable size on disk, all sstables are the same size)
+ *
+ * L3 [][][][][][][][][][][]
+ * L2 [ ][ ][ ][ ]
+ * L1 [ ][ ]
+ * L0 [ ]
+ *
+ * Will look like this after being dropped to L0 and sorted by last token (and, to illustrate overlap, the overlapping ones are put on a new line):
+ *
+ * [][][]
+ * [ ][][][]
+ * [ ]
+ * [ ]
+ *...
+ *
+ * Then, we start iterating from the smallest last-token and adding all sstables that do not cause an overlap to
+ * a level, we will reconstruct the original leveling top-down. Whenever we add an sstable to the level, we remove it from
+ * the sorted list. Once we reach the end of the sorted list, we have a full level, and can start over with the level below.
+ *
+ * If we end up with more levels than expected, we put all levels exceeding the expected in L0, for example, original L0
+ * files will most likely be put in a level of its own since they most often overlap many other sstables
+ */
+ public class SSTableOfflineRelevel
+ {
+ /**
+ * @param args a list of sstables whose metadata we are changing
+ */
+ public static void main(String[] args) throws IOException
+ {
+ PrintStream out = System.out;
+ if (args.length < 2)
+ {
+ out.println("This command should be run with Cassandra stopped!");
+ out.println("Usage: sstableofflinerelevel [--dry-run] <keyspace> <columnfamily>");
+ System.exit(1);
+ }
+ boolean dryRun = args[0].equals("--dry-run");
+ String keyspace = args[args.length - 2];
+ String columnfamily = args[args.length - 1];
- DatabaseDescriptor.loadSchemas();
++ DatabaseDescriptor.loadSchemas(false);
+
+ if (Schema.instance.getCFMetaData(keyspace, columnfamily) == null)
+ throw new IllegalArgumentException(String.format("Unknown keyspace/columnFamily %s.%s",
+ keyspace,
+ columnfamily));
+
- Keyspace.openWithoutSSTables(keyspace);
- Directories directories = Directories.create(keyspace, columnfamily);
++ Keyspace ks = Keyspace.openWithoutSSTables(keyspace);
++ ColumnFamilyStore cfs = ks.getColumnFamilyStore(columnfamily);
++ Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
+ Set<SSTableReader> sstables = new HashSet<>();
- for (Map.Entry<Descriptor, Set<Component>> sstable : directories.sstableLister().list().entrySet())
++ for (Map.Entry<Descriptor, Set<Component>> sstable : lister.list().entrySet())
+ {
+ if (sstable.getKey() != null)
+ {
+ SSTableReader reader = SSTableReader.open(sstable.getKey());
+ sstables.add(reader);
+ }
+ }
+ if (sstables.isEmpty())
+ {
+ out.println("No sstables to relevel for "+keyspace+"."+columnfamily);
+ System.exit(1);
+ }
+ Relevel rl = new Relevel(sstables);
+ rl.relevel(dryRun);
+ System.exit(0);
+
+ }
+
+ private static class Relevel
+ {
+ private final Set<SSTableReader> sstables;
+ private final int approxExpectedLevels;
+ public Relevel(Set<SSTableReader> sstables)
+ {
+ this.sstables = sstables;
+ approxExpectedLevels = (int) Math.ceil(Math.log10(sstables.size()));
+ }
+
+ public void relevel(boolean dryRun) throws IOException
+ {
+ List<SSTableReader> sortedSSTables = new ArrayList<>(sstables);
+ Collections.sort(sortedSSTables, new Comparator<SSTableReader>()
+ {
+ @Override
+ public int compare(SSTableReader o1, SSTableReader o2)
+ {
+ return o1.last.compareTo(o2.last);
+ }
+ });
+
+ List<List<SSTableReader>> levels = new ArrayList<>();
+
+ while (!sortedSSTables.isEmpty())
+ {
+ Iterator<SSTableReader> it = sortedSSTables.iterator();
+ List<SSTableReader> level = new ArrayList<>();
+ DecoratedKey lastLast = null;
+ while (it.hasNext())
+ {
+ SSTableReader sstable = it.next();
+ if (lastLast == null || lastLast.compareTo(sstable.first) < 0)
+ {
+ level.add(sstable);
+ lastLast = sstable.last;
+ it.remove();
+ }
+ }
+ levels.add(level);
+ }
+ List<SSTableReader> l0 = new ArrayList<>();
+ if (approxExpectedLevels < levels.size())
+ {
+ for (int i = approxExpectedLevels; i < levels.size(); i++)
+ l0.addAll(levels.get(i));
+ levels = levels.subList(0, approxExpectedLevels);
+ }
+ if (dryRun)
+ System.out.println("Potential leveling: ");
+ else
+ System.out.println("New leveling: ");
+
+ System.out.println("L0="+l0.size());
+ for (int i = levels.size() - 1; i >= 0; i--)
+ System.out.println(String.format("L%d %d", levels.size() - i, levels.get(i).size()));
+
+ if (!dryRun)
+ {
+ for (SSTableReader sstable : l0)
+ {
+ if (sstable.getSSTableLevel() != 0)
- LeveledManifest.mutateLevel(Pair.create(sstable.getSSTableMetadata(), sstable.getAncestors()), sstable.descriptor, sstable.descriptor.filenameFor(Component.STATS), 0);
++ sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 0);
+ }
+ for (int i = levels.size() - 1; i >= 0; i--)
+ {
+ for (SSTableReader sstable : levels.get(i))
+ {
+ int newLevel = levels.size() - i;
+ if (newLevel != sstable.getSSTableLevel())
- LeveledManifest.mutateLevel(Pair.create(sstable.getSSTableMetadata(), sstable.getAncestors()), sstable.descriptor, sstable.descriptor.filenameFor(Component.STATS), newLevel);
++ sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, newLevel);
+ }
+ }
+ }
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/cassandra/blob/1e74dd0d/tools/bin/sstableofflinerelevel
----------------------------------------------------------------------
diff --cc tools/bin/sstableofflinerelevel
index 0000000,19cb4c1..bfe28a0
mode 000000,100755..100755
--- a/tools/bin/sstableofflinerelevel
+++ b/tools/bin/sstableofflinerelevel
@@@ -1,0 -1,49 +1,54 @@@
+ #!/bin/sh
+
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+ if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
- for include in "`dirname $0`/cassandra.in.sh" \
- "$HOME/.cassandra.in.sh" \
- /usr/share/cassandra/cassandra.in.sh \
++ for include in /usr/share/cassandra/cassandra.in.sh \
+ /usr/local/share/cassandra/cassandra.in.sh \
- /opt/cassandra/cassandra.in.sh; do
- if [ -r $include ]; then
- . $include
++ /opt/cassandra/cassandra.in.sh \
++ ~/.cassandra.in.sh \
++ "`dirname "$0"`/cassandra.in.sh"; do
++ if [ -r "$include" ]; then
++ . "$include"
+ break
+ fi
+ done
-elif [ -r $CASSANDRA_INCLUDE ]; then
- . $CASSANDRA_INCLUDE
++elif [ -r "$CASSANDRA_INCLUDE" ]; then
++ . "$CASSANDRA_INCLUDE"
+ fi
+
-
+ # Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x $JAVA_HOME/bin/java ]; then
- JAVA=$JAVA_HOME/bin/java
++if [ -x "$JAVA_HOME/bin/java" ]; then
++ JAVA="$JAVA_HOME/bin/java"
+ else
- JAVA=`which java`
++ JAVA="`which java`"
+ fi
+
-if [ -z $CLASSPATH ]; then
++if [ -z "$CLASSPATH" ]; then
+ echo "You must set the CLASSPATH var" >&2
+ exit 1
+ fi
+
-$JAVA -cp $CLASSPATH -Dstorage-config=$CASSANDRA_CONF \
- -Dlog4j.configuration=log4j-tools.properties \
++if [ "x$MAX_HEAP_SIZE" = "x" ]; then
++ MAX_HEAP_SIZE="256M"
++fi
++
++"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" -Xmx$MAX_HEAP_SIZE \
++ -Dcassandra.storagedir="$cassandra_storagedir" \
++ -Dlogback.configurationFile=logback-tools.xml \
+ org.apache.cassandra.tools.SSTableOfflineRelevel "$@"
++