You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2015/03/02 12:51:03 UTC
cassandra git commit: Tool to create a leveling based on an existing
set of sstables.
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.0 5654e7368 -> 33a3a09cb
Tool to create a leveling based on an existing set of sstables.
Patch by marcuse; reviewed by carlyeks for CASSANDRA-8301
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/33a3a09c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/33a3a09c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/33a3a09c
Branch: refs/heads/cassandra-2.0
Commit: 33a3a09cbf2f0a31a823b6ef8f5e9fcc1546c9f5
Parents: 5654e73
Author: Marcus Eriksson <ma...@apache.org>
Authored: Wed Dec 17 21:50:38 2014 +0100
Committer: Marcus Eriksson <ma...@apache.org>
Committed: Mon Mar 2 11:41:25 2015 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../cassandra/tools/SSTableOfflineRelevel.java | 191 +++++++++++++++++++
tools/bin/sstableofflinerelevel | 49 +++++
3 files changed, 241 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/33a3a09c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3a8d824..f98bb3f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.13:
+ * Add offline tool to relevel sstables (CASSANDRA-8301)
* Preserve stream ID for more protocol errors (CASSANDRA-8848)
* Fix combining token() function with multi-column relations on
clustering columns (CASSANDRA-8797)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/33a3a09c/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
new file mode 100644
index 0000000..3fb2f7a
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.LeveledManifest;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * Create a decent leveling for the given keyspace/column family
+ *
+ * Approach is to sort the sstables by their last token
+ *
+ * given an original leveling like this (note that [ ] indicates token boundaries, not sstable size on disk, all sstables are the same size)
+ *
+ * L3 [][][][][][][][][][][]
+ * L2 [ ][ ][ ][ ]
+ * L1 [ ][ ]
+ * L0 [ ]
+ *
+ * Will look like this after being dropped to L0 and sorted by last token (and, to illustrate overlap, the overlapping ones are put on a new line):
+ *
+ * [][][]
+ * [ ][][][]
+ * [ ]
+ * [ ]
+ *...
+ *
+ * Then, we start iterating from the smallest last-token and adding all sstables that do not cause an overlap to
+ * a level, we will reconstruct the original leveling top-down. Whenever we add an sstable to the level, we remove it from
+ * the sorted list. Once we reach the end of the sorted list, we have a full level, and can start over with the level below.
+ *
+ * If we end up with more levels than expected, we put all levels exceeding the expected in L0, for example, original L0
+ * files will most likely be put in a level of its own since they most often overlap many other sstables
+ */
+public class SSTableOfflineRelevel
+{
+ /**
+ * @param args a list of sstables whose metadata we are changing
+ */
+ public static void main(String[] args) throws IOException
+ {
+ PrintStream out = System.out;
+ if (args.length < 2)
+ {
+ out.println("This command should be run with Cassandra stopped!");
+ out.println("Usage: sstableofflinerelevel [--dry-run] <keyspace> <columnfamily>");
+ System.exit(1);
+ }
+ boolean dryRun = args[0].equals("--dry-run");
+ String keyspace = args[args.length - 2];
+ String columnfamily = args[args.length - 1];
+ DatabaseDescriptor.loadSchemas();
+
+ if (Schema.instance.getCFMetaData(keyspace, columnfamily) == null)
+ throw new IllegalArgumentException(String.format("Unknown keyspace/columnFamily %s.%s",
+ keyspace,
+ columnfamily));
+
+ Keyspace.openWithoutSSTables(keyspace);
+ Directories directories = Directories.create(keyspace, columnfamily);
+ Set<SSTableReader> sstables = new HashSet<>();
+ for (Map.Entry<Descriptor, Set<Component>> sstable : directories.sstableLister().list().entrySet())
+ {
+ if (sstable.getKey() != null)
+ {
+ SSTableReader reader = SSTableReader.open(sstable.getKey());
+ sstables.add(reader);
+ }
+ }
+ if (sstables.isEmpty())
+ {
+ out.println("No sstables to relevel for "+keyspace+"."+columnfamily);
+ System.exit(1);
+ }
+ Relevel rl = new Relevel(sstables);
+ rl.relevel(dryRun);
+ System.exit(0);
+
+ }
+
+ private static class Relevel
+ {
+ private final Set<SSTableReader> sstables;
+ private final int approxExpectedLevels;
+ public Relevel(Set<SSTableReader> sstables)
+ {
+ this.sstables = sstables;
+ approxExpectedLevels = (int) Math.ceil(Math.log10(sstables.size()));
+ }
+
+ public void relevel(boolean dryRun) throws IOException
+ {
+ List<SSTableReader> sortedSSTables = new ArrayList<>(sstables);
+ Collections.sort(sortedSSTables, new Comparator<SSTableReader>()
+ {
+ @Override
+ public int compare(SSTableReader o1, SSTableReader o2)
+ {
+ return o1.last.compareTo(o2.last);
+ }
+ });
+
+ List<List<SSTableReader>> levels = new ArrayList<>();
+
+ while (!sortedSSTables.isEmpty())
+ {
+ Iterator<SSTableReader> it = sortedSSTables.iterator();
+ List<SSTableReader> level = new ArrayList<>();
+ DecoratedKey lastLast = null;
+ while (it.hasNext())
+ {
+ SSTableReader sstable = it.next();
+ if (lastLast == null || lastLast.compareTo(sstable.first) < 0)
+ {
+ level.add(sstable);
+ lastLast = sstable.last;
+ it.remove();
+ }
+ }
+ levels.add(level);
+ }
+ List<SSTableReader> l0 = new ArrayList<>();
+ if (approxExpectedLevels < levels.size())
+ {
+ for (int i = approxExpectedLevels; i < levels.size(); i++)
+ l0.addAll(levels.get(i));
+ levels = levels.subList(0, approxExpectedLevels);
+ }
+ if (dryRun)
+ System.out.println("Potential leveling: ");
+ else
+ System.out.println("New leveling: ");
+
+ System.out.println("L0="+l0.size());
+ for (int i = levels.size() - 1; i >= 0; i--)
+ System.out.println(String.format("L%d %d", levels.size() - i, levels.get(i).size()));
+
+ if (!dryRun)
+ {
+ for (SSTableReader sstable : l0)
+ {
+ if (sstable.getSSTableLevel() != 0)
+ LeveledManifest.mutateLevel(Pair.create(sstable.getSSTableMetadata(), sstable.getAncestors()), sstable.descriptor, sstable.descriptor.filenameFor(Component.STATS), 0);
+ }
+ for (int i = levels.size() - 1; i >= 0; i--)
+ {
+ for (SSTableReader sstable : levels.get(i))
+ {
+ int newLevel = levels.size() - i;
+ if (newLevel != sstable.getSSTableLevel())
+ LeveledManifest.mutateLevel(Pair.create(sstable.getSSTableMetadata(), sstable.getAncestors()), sstable.descriptor, sstable.descriptor.filenameFor(Component.STATS), newLevel);
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/33a3a09c/tools/bin/sstableofflinerelevel
----------------------------------------------------------------------
diff --git a/tools/bin/sstableofflinerelevel b/tools/bin/sstableofflinerelevel
new file mode 100755
index 0000000..19cb4c1
--- /dev/null
+++ b/tools/bin/sstableofflinerelevel
@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
+ for include in "`dirname $0`/cassandra.in.sh" \
+ "$HOME/.cassandra.in.sh" \
+ /usr/share/cassandra/cassandra.in.sh \
+ /usr/local/share/cassandra/cassandra.in.sh \
+ /opt/cassandra/cassandra.in.sh; do
+ if [ -r $include ]; then
+ . $include
+ break
+ fi
+ done
+elif [ -r $CASSANDRA_INCLUDE ]; then
+ . $CASSANDRA_INCLUDE
+fi
+
+
+# Use JAVA_HOME if set, otherwise look for java in PATH
+if [ -x $JAVA_HOME/bin/java ]; then
+ JAVA=$JAVA_HOME/bin/java
+else
+ JAVA=`which java`
+fi
+
+if [ -z $CLASSPATH ]; then
+ echo "You must set the CLASSPATH var" >&2
+ exit 1
+fi
+
+$JAVA -cp $CLASSPATH -Dstorage-config=$CASSANDRA_CONF \
+ -Dlog4j.configuration=log4j-tools.properties \
+ org.apache.cassandra.tools.SSTableOfflineRelevel "$@"