You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2012/05/04 01:35:43 UTC
[3/10] git commit: merge from 1.0
merge from 1.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/af1dc0f5
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/af1dc0f5
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/af1dc0f5
Branch: refs/heads/cassandra-1.1
Commit: af1dc0f56a560ddee1e98bfea677bf534d38cc39
Parents: 249456b c1148ce
Author: Jonathan Ellis <jb...@apache.org>
Authored: Thu May 3 18:31:29 2012 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu May 3 18:31:29 2012 -0500
----------------------------------------------------------------------
CHANGES.txt | 9 +++-
src/java/org/apache/cassandra/db/Directories.java | 33 ++++----------
.../cassandra/db/compaction/CompactionTask.java | 20 ++-------
.../apache/cassandra/io/sstable/Descriptor.java | 15 ++++++-
.../cassandra/io/sstable/SSTableMetadata.java | 4 +-
5 files changed, 38 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index b13d69b,1c1c184..1b86758
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,121 -1,19 +1,128 @@@
-1.0.10
+1.1.1-dev
+ * include tombstone size in ColumnFamily.size, which can prevent OOM
+ during sudden mass delete operations (CASSANDRA-3741)
+ * Open 1 sstableScanner per level for leveled compaction (CASSANDRA-4142)
+ * Optimize reads when row deletion timestamps allow us to restrict
+ the set of sstables we check (CASSANDRA-4116)
+ * incremental repair by token range (CASSANDRA-3912)
+ * streaming commitlog backup + pitr (CASSANDRA-3690)
+ * avoid generating redundant compaction tasks during streaming
+ (CASSANDRA-4174)
+ * add -cf option to nodetool snapshot, and takeColumnFamilySnapshot to
+ StorageService mbean (CASSANDRA-556)
+ * optimize cleanup to drop entire sstables where possible (CASSANDRA-4079)
+ * optimize truncate when autosnapshot is disabled (CASSANDRA-4153)
+ * add support for commitlog archiving and point-in-time recovery
+ (CASSANDRA-3647)
+ * update caches to use byte[] keys to reduce memory overhead (CASSANDRA-3966)
+ * add column limit to cli (CASSANDRA-3012, 4098)
+ * clean up and optimize DataOutputBuffer, used by CQL compression and
+ CompositeType (CASSANDRA-4072)
+ * optimize commitlog checksumming (CASSANDRA-3610)
+ * identify and blacklist corrupted SSTables from future compactions
+ (CASSANDRA-2261)
+ * Move CfDef and KsDef validation out of thrift (CASSANDRA-4037)
+ * Expose repairing by a user provided range (CASSANDRA-3912)
+ * Add way to force the cassandra-cli to refresh it's schema (CASSANDRA-4052)
+ * Avoids having replicate on write tasks stacking up at CL.ONE (CASSANDRA-2889)
+ * (cql3) Fix order by for reversed queries (CASSANDRA-4160)
+ * (cql3) Add ReversedType support (CASSANDRA-4004)
+ * (cql3) Add timeuuid type (CASSANDRA-4194)
+ * (cql3) Minor fixes (CASSANDRA-4185)
+ * (cql3) Fix prepared statement in BATCH (CASSANDRA-4202)
+ * (cql3) Reduce the list of reserved keywords (CASSANDRA-4186)
+ * (cql3) Move max/min compaction thresholds to compaction strategy options
+ (CASSANDRA-4187)
+Merged from 1.0:
+ * Fix super columns bug where cache is not updated (CASSANDRA-4190)
* fix maxTimestamp to include row tombstones (CASSANDRA-4116)
+ * (CLI) properly handle quotes in create/update keyspace commands (CASSANDRA-4129)
+ * Avoids possible deadlock during bootstrap (CASSANDRA-4159)
+ * fix stress tool that hangs forever on timeout or error (CASSANDRA-4128)
+ * Fix super columns bug where cache is not updated (CASSANDRA-4190)
+ * stress tool to return appropriate exit code on failure (CASSANDRA-4188)
++ * fix compaction NPE when out of disk space and assertions disabled
++ (CASSANDRA-3985)
+
+
+1.1.0-final
+ * average a reduced liveRatio estimate with the previous one (CASSANDRA-4065)
+ * Allow KS and CF names up to 48 characters (CASSANDRA-4157)
+ * fix stress build (CASSANDRA-4140)
+ * add time remaining estimate to nodetool compactionstats (CASSANDRA-4167)
+ * (cql) fix NPE in cql3 ALTER TABLE (CASSANDRA-4163)
+ * (cql) Add support for CL.TWO and CL.THREE in CQL (CASSANDRA-4156)
+ * (cql) Fix type in CQL3 ALTER TABLE preventing update (CASSANDRA-4170)
+ * (cql) Throw invalid exception from CQL3 on obsolete options (CASSANDRA-4171)
+ * (cqlsh) fix recognizing uppercase SELECT keyword (CASSANDRA-4161)
+ * Pig: wide row support (CASSANDRA-3909)
+Merged from 1.0:
* avoid streaming empty files with bulk loader if sstablewriter errors out
(CASSANDRA-3946)
+
+
+1.1-rc1
+ * Include stress tool in binary builds (CASSANDRA-4103)
+ * (Hadoop) fix wide row iteration when last row read was deleted
+ (CASSANDRA-4154)
+ * fix read_repair_chance to really default to 0.1 in the cli (CASSANDRA-4114)
+ * Adds caching and bloomFilterFpChange to CQL options (CASSANDRA-4042)
+ * Adds posibility to autoconfigure size of the KeyCache (CASSANDRA-4087)
+ * fix KEYS index from skipping results (CASSANDRA-3996)
+ * Remove sliced_buffer_size_in_kb dead option (CASSANDRA-4076)
+ * make loadNewSStable preserve sstable version (CASSANDRA-4077)
+ * Respect 1.0 cache settings as much as possible when upgrading
+ (CASSANDRA-4088)
+ * relax path length requirement for sstable files when upgrading on
+ non-Windows platforms (CASSANDRA-4110)
+ * fix terminination of the stress.java when errors were encountered
+ (CASSANDRA-4128)
+ * Move CfDef and KsDef validation out of thrift (CASSANDRA-4037)
+ * Fix get_paged_slice (CASSANDRA-4136)
+ * CQL3: Support slice with exclusive start and stop (CASSANDRA-3785)
+Merged from 1.0:
* support PropertyFileSnitch in bulk loader (CASSANDRA-4145)
* add auto_snapshot option allowing disabling snapshot before drop/truncate
(CASSANDRA-3710)
* allow short snitch names (CASSANDRA-4130)
+
+
+1.1-beta2
+ * rename loaded sstables to avoid conflicts with local snapshots
+ (CASSANDRA-3967)
+ * start hint replay as soon as FD notifies that the target is back up
+ (CASSANDRA-3958)
+ * avoid unproductive deserializing of cached rows during compaction
+ (CASSANDRA-3921)
+ * fix concurrency issues with CQL keyspace creation (CASSANDRA-3903)
+ * Show Effective Owership via Nodetool ring <keyspace> (CASSANDRA-3412)
+ * Update ORDER BY syntax for CQL3 (CASSANDRA-3925)
+ * Fix BulkRecordWriter to not throw NPE if reducer gets no map data from Hadoop (CASSANDRA-3944)
+ * Fix bug with counters in super columns (CASSANDRA-3821)
+ * Remove deprecated merge_shard_chance (CASSANDRA-3940)
+ * add a convenient way to reset a node's schema (CASSANDRA-2963)
+ * fix for intermittent SchemaDisagreementException (CASSANDRA-3884)
+ * CLI `list <CF>` to limit number of columns and their order (CASSANDRA-3012)
+ * ignore deprecated KsDef/CfDef/ColumnDef fields in native schema (CASSANDRA-3963)
+ * CLI to report when unsupported column_metadata pair was given (CASSANDRA-3959)
+ * reincarnate removed and deprecated KsDef/CfDef attributes (CASSANDRA-3953)
+ * Fix race between writes and read for cache (CASSANDRA-3862)
+ * perform static initialization of StorageProxy on start-up (CASSANDRA-3797)
+ * support trickling fsync() on writes (CASSANDRA-3950)
+ * expose counters for unavailable/timeout exceptions given to thrift clients (CASSANDRA-3671)
+ * avoid quadratic startup time in LeveledManifest (CASSANDRA-3952)
+ * Add type information to new schema_ columnfamilies and remove thrift
+ serialization for schema (CASSANDRA-3792)
+ * add missing column validator options to the CLI help (CASSANDRA-3926)
+ * skip reading saved key cache if CF's caching strategy is NONE or ROWS_ONLY (CASSANDRA-3954)
+ * Unify migration code (CASSANDRA-4017)
+Merged from 1.0:
* cqlsh: guess correct version of Python for Arch Linux (CASSANDRA-4090)
+ * (CLI) properly handle quotes in create/update keyspace commands (CASSANDRA-4129)
+ * Avoids possible deadlock during bootstrap (CASSANDRA-4159)
+ * fix stress tool that hangs forever on timeout or error (CASSANDRA-4128)
+ * Fix super columns bug where cache is not updated (CASSANDRA-4190)
+ * stress tool to return appropriate exit code on failure (CASSANDRA-4188)
- * fix compaction NPE when out of disk space and assertions disabled
- (CASSANDRA-3985)
1.0.9
http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/Directories.java
index 6d16559,0000000..f023c8c
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@@ -1,584 -1,0 +1,571 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOError;
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.commons.lang.StringUtils;
+import com.google.common.collect.ImmutableMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.*;
+import org.apache.cassandra.db.compaction.LeveledManifest;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.MmappedSegmentedFile;
+import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.CLibrary;
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * Encapsulate handling of paths to the data files.
+ *
+ * The directory layout is the following:
+ * /<path_to_data_dir>/ks/cf1/ks-cf1-hb-1-Data.db
+ * /cf2/ks-cf2-hb-1-Data.db
+ * ...
+ *
+ * In addition, more that one 'root' data directory can be specified so that
+ * <path_to_data_dir> potentially represents multiple locations.
+ * Note that in the case of multiple locations, the manifest for the leveled
+ * compaction is only in one of the location.
+ *
+ * Snapshots (resp. backups) are always created along the sstables thare are
+ * snapshoted (resp. backuped) but inside a subdirectory named 'snapshots'
+ * (resp. backups) (and snapshots are furter inside a subdirectory of the name
+ * of the snapshot).
+ *
+ * This class abstracts all those details from the rest of the code.
+ */
+public class Directories
+{
+ private static Logger logger = LoggerFactory.getLogger(Directories.class);
+
+ public static final String BACKUPS_SUBDIR = "backups";
+ public static final String SNAPSHOT_SUBDIR = "snapshots";
+ public static final char SECONDARY_INDEX_NAME_SEPARATOR = '.';
+
+ public static final File[] dataFileLocations;
+ static
+ {
+ String[] locations = DatabaseDescriptor.getAllDataFileLocations();
+ dataFileLocations = new File[locations.length];
+ for (int i = 0; i < locations.length; ++i)
+ dataFileLocations[i] = new File(locations[i]);
+ }
+
+ private final String tablename;
+ private final String cfname;
+ private final File[] sstableDirectories;
+
+ public static Directories create(String tablename, String cfname)
+ {
+ int idx = cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
+ if (idx > 0)
+ // secondary index, goes in the same directory than the base cf
+ return new Directories(tablename, cfname, cfname.substring(0, idx));
+ else
+ return new Directories(tablename, cfname, cfname);
+ }
+
+ private Directories(String tablename, String cfname, String directoryName)
+ {
+ this.tablename = tablename;
+ this.cfname = cfname;
+ this.sstableDirectories = new File[dataFileLocations.length];
+ for (int i = 0; i < dataFileLocations.length; ++i)
+ sstableDirectories[i] = new File(dataFileLocations[i], join(tablename, directoryName));
+
+ if (!StorageService.instance.isClientMode())
+ {
+ try
+ {
+ for (File dir : sstableDirectories)
+ FileUtils.createDirectory(dir);
+ }
+ catch (IOException e)
+ {
+ throw new IOError(e);
+ }
+ }
+ }
+
+ public File getDirectoryForNewSSTables(long estimatedSize)
+ {
- return getDirectoryForNewSSTables(estimatedSize, true);
- }
++ File path = getLocationWithMaximumAvailableSpace(estimatedSize);
+
- public File getDirectoryForNewSSTables(long estimatedSize, boolean ensureFreeSpace)
- {
- File path = getLocationWithMaximumAvailableSpace(estimatedSize, ensureFreeSpace);
+ // Requesting GC has a chance to free space only if we're using mmap and a non SUN jvm
+ if (path == null
- && (DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap || DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap)
- && !MmappedSegmentedFile.isCleanerAvailable())
++ && (DatabaseDescriptor.getDiskAccessMode() == Config.DiskAccessMode.mmap || DatabaseDescriptor.getIndexAccessMode() == Config.DiskAccessMode.mmap)
++ && !MmappedSegmentedFile.isCleanerAvailable())
+ {
++ logger.info("Forcing GC to free up disk space. Upgrade to the Oracle JVM to avoid this");
+ StorageService.instance.requestGC();
+ // retry after GCing has forced unmap of compacted SSTables so they can be deleted
+ // Note: GCInspector will do this already, but only sun JVM supports GCInspector so far
+ SSTableDeletingTask.rescheduleFailedTasks();
+ try
+ {
+ Thread.sleep(10000);
+ }
+ catch (InterruptedException e)
+ {
+ throw new AssertionError(e);
+ }
- path = getLocationWithMaximumAvailableSpace(estimatedSize, ensureFreeSpace);
++ path = getLocationWithMaximumAvailableSpace(estimatedSize);
+ }
++
+ return path;
+ }
+
+ /*
+ * Loop through all the disks to see which disk has the max free space
+ * return the disk with max free space for compactions. If the size of the expected
+ * compacted file is greater than the max disk space available return null, we cannot
+ * do compaction in this case.
+ */
- public File getLocationWithMaximumAvailableSpace(long estimatedSize, boolean ensureFreeSpace)
++ public File getLocationWithMaximumAvailableSpace(long estimatedSize)
+ {
+ long maxFreeDisk = 0;
+ File maxLocation = null;
+
+ for (File dir : sstableDirectories)
+ {
+ if (maxFreeDisk < dir.getUsableSpace())
+ {
+ maxFreeDisk = dir.getUsableSpace();
+ maxLocation = dir;
+ }
+ }
- logger.debug(String.format("expected data files size is %d; largest free partition (%s) has %d bytes free",
- estimatedSize,
- maxLocation,
- maxFreeDisk));
-
+ // Load factor of 0.9 we do not want to use the entire disk that is too risky.
+ maxFreeDisk = (long) (0.9 * maxFreeDisk);
++ logger.debug(String.format("expected data files size is %d; largest free partition (%s) has %d bytes free",
++ estimatedSize, maxLocation, maxFreeDisk));
+
- if (!ensureFreeSpace || estimatedSize < maxFreeDisk)
- {
- if (estimatedSize >= maxFreeDisk)
- logger.warn(String.format("Data file location %s only has %d free, estimated size is %d",
- maxLocation,
- maxFreeDisk,
- estimatedSize));
+
++ if (estimatedSize < maxFreeDisk)
+ return maxLocation;
- }
-
+ return null;
+ }
+
+ public static File getSnapshotDirectory(Descriptor desc, String snapshotName)
+ {
+ return getOrCreate(desc.directory, SNAPSHOT_SUBDIR, snapshotName);
+ }
+
+ public static File getBackupsDirectory(Descriptor desc)
+ {
+ return getOrCreate(desc.directory, BACKUPS_SUBDIR);
+ }
+
+ public SSTableLister sstableLister()
+ {
+ return new SSTableLister();
+ }
+
+ public class SSTableLister
+ {
+ private boolean skipCompacted;
+ private boolean skipTemporary;
+ private boolean includeBackups;
+ private boolean onlyBackups;
+ private int nbFiles;
+ private final Map<Descriptor, Set<Component>> components = new HashMap<Descriptor, Set<Component>>();
+ private boolean filtered;
+ private String snapshotName;
+
+ public SSTableLister skipCompacted(boolean b)
+ {
+ if (filtered)
+ throw new IllegalStateException("list() has already been called");
+ skipCompacted = b;
+ return this;
+ }
+
+ public SSTableLister skipTemporary(boolean b)
+ {
+ if (filtered)
+ throw new IllegalStateException("list() has already been called");
+ skipTemporary = b;
+ return this;
+ }
+
+ public SSTableLister includeBackups(boolean b)
+ {
+ if (filtered)
+ throw new IllegalStateException("list() has already been called");
+ includeBackups = b;
+ return this;
+ }
+
+ public SSTableLister onlyBackups(boolean b)
+ {
+ if (filtered)
+ throw new IllegalStateException("list() has already been called");
+ onlyBackups = b;
+ includeBackups = b;
+ return this;
+ }
+
+ public SSTableLister snapshots(String sn)
+ {
+ if (filtered)
+ throw new IllegalStateException("list() has already been called");
+ snapshotName = sn;
+ return this;
+ }
+
+ public Map<Descriptor, Set<Component>> list()
+ {
+ filter();
+ return ImmutableMap.copyOf(components);
+ }
+
+ public List<File> listFiles()
+ {
+ filter();
+ List<File> l = new ArrayList<File>(nbFiles);
+ for (Map.Entry<Descriptor, Set<Component>> entry : components.entrySet())
+ {
+ for (Component c : entry.getValue())
+ {
+ l.add(new File(entry.getKey().filenameFor(c)));
+ }
+ }
+ return l;
+ }
+
+ private void filter()
+ {
+ if (filtered)
+ return;
+
+ for (File location : sstableDirectories)
+ {
+ if (snapshotName != null)
+ {
+ new File(location, join(SNAPSHOT_SUBDIR, snapshotName)).listFiles(getFilter());
+ continue;
+ }
+
+ if (!onlyBackups)
+ location.listFiles(getFilter());
+
+ if (includeBackups)
+ new File(location, BACKUPS_SUBDIR).listFiles(getFilter());
+ }
+ filtered = true;
+ }
+
+ private FileFilter getFilter()
+ {
+ // Note: the prefix needs to include cfname + separator to distinguish between a cfs and it's secondary indexes
+ final String sstablePrefix = tablename + Component.separator + cfname + Component.separator;
+ return new FileFilter()
+ {
+ // This function always return false since accepts adds to the components map
+ public boolean accept(File file)
+ {
+ // we are only interested in the SSTable files that belong to the specific ColumnFamily
+ if (file.isDirectory() || !file.getName().startsWith(sstablePrefix))
+ return false;
+
+ Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
+ if (pair == null)
+ return false;
+
+ if (skipCompacted && new File(pair.left.filenameFor(Component.COMPACTED_MARKER)).exists())
+ return false;
+ if (skipTemporary && pair.left.temporary)
+ return false;
+
+ Set<Component> previous = components.get(pair.left);
+ if (previous == null)
+ {
+ previous = new HashSet<Component>();
+ components.put(pair.left, previous);
+ }
+ previous.add(pair.right);
+ nbFiles++;
+ return false;
+ }
+ };
+ }
+ }
+
+ public File tryGetLeveledManifest()
+ {
+ for (File dir : sstableDirectories)
+ {
+ File manifestFile = new File(dir, cfname + LeveledManifest.EXTENSION);
+ if (manifestFile.exists())
+ {
+ logger.debug("Found manifest at {}", manifestFile);
+ return manifestFile;
+ }
+ }
+ logger.debug("No level manifest found");
+ return null;
+ }
+
+ public File getOrCreateLeveledManifest()
+ {
+ File manifestFile = tryGetLeveledManifest();
+ if (manifestFile == null)
+ manifestFile = new File(sstableDirectories[0], cfname + LeveledManifest.EXTENSION);
+ return manifestFile;
+ }
+
+ public void snapshotLeveledManifest(String snapshotName) throws IOException
+ {
+ File manifest = tryGetLeveledManifest();
+ if (manifest != null)
+ {
+ File snapshotDirectory = getOrCreate(manifest.getParentFile(), SNAPSHOT_SUBDIR, snapshotName);
+ CLibrary.createHardLink(manifest, new File(snapshotDirectory, manifest.getName()));
+ }
+ }
+
+ public boolean snapshotExists(String snapshotName)
+ {
+ for (File dir : sstableDirectories)
+ {
+ File snapshotDir = new File(dir, join(SNAPSHOT_SUBDIR, snapshotName));
+ if (snapshotDir.exists())
+ return true;
+ }
+ return false;
+ }
+
+ public void clearSnapshot(String snapshotName) throws IOException
+ {
+ // If snapshotName is empty or null, we will delete the entire snapshot directory
+ String tag = snapshotName == null ? "" : snapshotName;
+ for (File dir : sstableDirectories)
+ {
+ File snapshotDir = new File(dir, join(SNAPSHOT_SUBDIR, tag));
+ if (snapshotDir.exists())
+ {
+ if (logger.isDebugEnabled())
+ logger.debug("Removing snapshot directory " + snapshotDir);
+ FileUtils.deleteRecursive(snapshotDir);
+ }
+ }
+ }
+
+ private static File getOrCreate(File base, String... subdirs)
+ {
+ File dir = subdirs == null || subdirs.length == 0 ? base : new File(base, join(subdirs));
+ if (dir.exists())
+ {
+ if (!dir.isDirectory())
+ throw new IOError(new IOException(String.format("Invalid directory path %s: path exists but is not a directory", dir)));
+ }
+ else if (!dir.mkdirs())
+ {
+ throw new IOError(new IOException("Unable to create directory " + dir));
+ }
+ return dir;
+ }
+
+ private static String join(String... s)
+ {
+ return StringUtils.join(s, File.separator);
+ }
+
+ /**
+ * To check if sstables needs migration, we look at the System directory.
+ * If it contains a directory for the status cf, we'll attempt a sstable
+ * migration.
+ * Note that it is mostly harmless to try a migration uselessly, except
+ * maybe for some wasted cpu cycles.
+ */
+ public static boolean sstablesNeedsMigration()
+ {
+ if (StorageService.instance.isClientMode())
+ return false;
+
+ boolean hasSystemKeyspace = false;
+ for (File location : dataFileLocations)
+ {
+ File systemDir = new File(location, Table.SYSTEM_TABLE);
+ hasSystemKeyspace |= (systemDir.exists() && systemDir.isDirectory());
+ File statusCFDir = new File(systemDir, SystemTable.STATUS_CF);
+ if (statusCFDir.exists())
+ return false;
+ }
+ if (!hasSystemKeyspace)
+ // This is a brand new node.
+ return false;
+
+ // Check whether the migration might create too long a filename
+ int longestLocation = -1;
+ try
+ {
+ for (File loc : dataFileLocations)
+ longestLocation = Math.max(longestLocation, loc.getCanonicalPath().length());
+ }
+ catch (IOException e)
+ {
+ throw new IOError(e);
+ }
+
+ // Check that migration won't error out halfway through from too-long paths. For Windows, we need to check
+ // total path length <= 255 (see http://msdn.microsoft.com/en-us/library/aa365247.aspx and discussion on CASSANDRA-2749);
+ // elsewhere, we just need to make sure filename is <= 255.
+ for (KSMetaData ksm : Schema.instance.getTableDefinitions())
+ {
+ String ksname = ksm.name;
+ for (Map.Entry<String, CFMetaData> entry : ksm.cfMetaData().entrySet())
+ {
+ String cfname = entry.getKey();
+
+ // max path is roughly (guess-estimate) <location>/ksname/cfname/snapshots/1324314347102-somename/ksname-cfname-tmp-hb-65536-Statistics.db
+ if (System.getProperty("os.name").startsWith("Windows")
+ && longestLocation + (ksname.length() + cfname.length()) * 2 + 63 > 255)
+ {
+ throw new RuntimeException(String.format("Starting with 1.1, keyspace names and column family " +
+ "names must be less than %s characters long. %s/%s doesn't" +
+ " respect that restriction. Please rename your " +
+ "keyspace/column families to respect that restriction " +
+ "before updating.", Schema.NAME_LENGTH, ksname, cfname));
+ }
+
+ if (ksm.name.length() + cfname.length() + 28 > 255)
+ {
+ throw new RuntimeException("Starting with 1.1, the keyspace name is included in data filenames. For "
+ + ksm.name + "/" + cfname + ", this puts you over the largest possible filename of 255 characters");
+ }
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Move sstables from the pre-#2749 layout to their new location/names.
+ * This involves:
+ * - moving each sstable to their CF specific directory
+ * - rename the sstable to include the keyspace in the filename
+ *
+ * Note that this also move leveled manifests, snapshots and backups.
+ */
+ public static void migrateSSTables()
+ {
+ logger.info("Upgrade from pre-1.1 version detected: migrating sstables to new directory layout");
+
+ for (File location : dataFileLocations)
+ {
+ if (!location.exists() || !location.isDirectory())
+ continue;
+
+ for (File ksDir : location.listFiles())
+ {
+ if (!ksDir.isDirectory())
+ continue;
+
+ for (File file : ksDir.listFiles())
+ migrateFile(file, ksDir, null);
+
+ migrateSnapshots(ksDir);
+ migrateBackups(ksDir);
+ }
+ }
+ }
+
+ private static void migrateSnapshots(File ksDir)
+ {
+ File snapshotDir = new File(ksDir, SNAPSHOT_SUBDIR);
+ if (!snapshotDir.exists())
+ return;
+
+ for (File snapshot : snapshotDir.listFiles())
+ {
+ if (!snapshot.isDirectory())
+ continue;
+
+ for (File f : snapshot.listFiles())
+ migrateFile(f, ksDir, join(SNAPSHOT_SUBDIR, snapshot.getName()));
+
+ if (!snapshot.delete())
+ logger.info("Old snapsot directory {} not deleted by migraation as it is not empty", snapshot);
+ }
+ if (!snapshotDir.delete())
+ logger.info("Old directory {} not deleted by migration as it is not empty", snapshotDir);
+ }
+
+ private static void migrateBackups(File ksDir)
+ {
+ File backupDir = new File(ksDir, BACKUPS_SUBDIR);
+ if (!backupDir.exists())
+ return;
+
+ for (File f : backupDir.listFiles())
+ migrateFile(f, ksDir, BACKUPS_SUBDIR);
+
+ if (!backupDir.delete())
+ logger.info("Old directory {} not deleted by migration as it is not empty", backupDir);
+ }
+
+ private static void migrateFile(File file, File ksDir, String additionalPath)
+ {
+ try
+ {
+ if (file.isDirectory())
+ return;
+
+ String name = file.getName();
+ boolean isManifest = name.endsWith(LeveledManifest.EXTENSION);
+ String cfname = isManifest
+ ? name.substring(0, name.length() - LeveledManifest.EXTENSION.length())
+ : name.substring(0, name.indexOf(Component.separator));
+
+ int idx = cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR); // idx > 0 => secondary index
+ String dirname = idx > 0 ? cfname.substring(0, idx) : cfname;
+ File destDir = getOrCreate(ksDir, dirname, additionalPath);
+
+ File destFile = new File(destDir, isManifest ? name : ksDir.getName() + Component.separator + name);
+ logger.debug(String.format("[upgrade to 1.1] Moving %s to %s", file, destFile));
+ FileUtils.renameWithConfirm(file, destFile);
+ }
+ catch (IOException e)
+ {
+ throw new IOError(e);
+ }
+ }
+
+ // Hack for tests, don't use otherwise
+ static void overrideDataDirectoriesForTest(String loc)
+ {
+ for (int i = 0; i < dataFileLocations.length; ++i)
+ dataFileLocations[i] = new File(loc);
+ }
+
+ // Hack for tests, don't use otherwise
+ static void resetDataDirectoriesAfterTest()
+ {
+ String[] locations = DatabaseDescriptor.getAllDataFileLocations();
+ for (int i = 0; i < locations.length; ++i)
+ dataFileLocations[i] = new File(locations[i]);
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index b66f20b,2a1b415..6d914e3
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@@ -18,10 -18,9 +18,9 @@@
package org.apache.cassandra.db.compaction;
+import java.io.File;
import java.io.IOException;
import java.util.*;
--import java.util.Map.Entry;
import com.google.common.base.Predicates;
import com.google.common.collect.Iterators;
@@@ -75,9 -76,9 +74,7 @@@ public class CompactionTask extends Abs
if (!isCompactionInteresting(toCompact))
return 0;
- File compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact),
- ensureFreeSpace());
-
- if (compactionFileLocation == null)
- compactionFileLocation = cfs.table.getDataFileLocation(cfs.getExpectedCompactedFileSize(toCompact));
-
++ File compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact));
if (compactionFileLocation == null && partialCompactionsAcceptable())
{
// If the compaction file path is null that means we have no space left for this compaction.
@@@ -88,17 -89,14 +85,15 @@@
// Note that we have removed files that are still marked as compacting.
// This suboptimal but ok since the caller will unmark all the sstables at the end.
toCompact.remove(cfs.getMaxSizeFile(toCompact));
- compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact),
- ensureFreeSpace());
- compactionFileLocation = cfs.table.getDataFileLocation(cfs.getExpectedCompactedFileSize(toCompact));
++ compactionFileLocation = cfs.directories.getDirectoryForNewSSTables(cfs.getExpectedCompactedFileSize(toCompact));
}
}
+
if (compactionFileLocation == null)
{
- logger.warn("insufficient space to compact even the two smallest files, aborting");
+ logger.warn("insufficient space to compact; aborting compaction");
return 0;
}
- assert compactionFileLocation != null;
if (DatabaseDescriptor.isSnapshotBeforeCompaction())
cfs.snapshotWithoutFlush(System.currentTimeMillis() + "-" + "compact-" + cfs.columnFamily);
@@@ -204,10 -198,10 +199,10 @@@
cfs.replaceCompactedSSTables(toCompact, sstables, compactionType);
// TODO: this doesn't belong here, it should be part of the reader to load when the tracker is wired up
-- for (Entry<SSTableReader, Map<DecoratedKey, Long>> ssTableReaderMapEntry : cachedKeyMap.entrySet())
++ for (Map.Entry<SSTableReader, Map<DecoratedKey, Long>> ssTableReaderMapEntry : cachedKeyMap.entrySet())
{
SSTableReader key = ssTableReaderMapEntry.getKey();
-- for (Entry<DecoratedKey, Long> entry : ssTableReaderMapEntry.getValue().entrySet())
++ for (Map.Entry<DecoratedKey, Long> entry : ssTableReaderMapEntry.getValue().entrySet())
key.cacheKey(entry.getKey(), entry.getValue());
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/src/java/org/apache/cassandra/io/sstable/Descriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/af1dc0f5/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
----------------------------------------------------------------------