You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by yu...@apache.org on 2014/01/07 17:31:27 UTC
git commit: CF id is now non-deterministic
Updated Branches:
refs/heads/trunk 5e304eb78 -> 0a1b277d6
CF id is now non-deterministic
and data dir/key cache are also unique per CF id.
patch by yukim; reviewed by Pavel Yaskevich for CASSANDRA-5202
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0a1b277d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0a1b277d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0a1b277d
Branch: refs/heads/trunk
Commit: 0a1b277d659e20cd19eaedaa7220b0fc55950dc4
Parents: 5e304eb
Author: Yuki Morishita <yu...@apache.org>
Authored: Tue Jan 7 10:28:48 2014 -0600
Committer: Yuki Morishita <yu...@apache.org>
Committed: Tue Jan 7 10:28:48 2014 -0600
----------------------------------------------------------------------
CHANGES.txt | 2 +
NEWS.txt | 11 ++-
.../apache/cassandra/cache/AutoSavingCache.java | 23 ++---
.../org/apache/cassandra/cache/CacheKey.java | 38 +++++++-
.../org/apache/cassandra/cache/KeyCacheKey.java | 22 +++--
.../org/apache/cassandra/cache/RowCacheKey.java | 5 +-
.../org/apache/cassandra/config/CFMetaData.java | 40 +++++++--
.../cassandra/config/DatabaseDescriptor.java | 13 ++-
.../apache/cassandra/db/ColumnFamilyStore.java | 26 +++---
.../org/apache/cassandra/db/Directories.java | 76 +++++++++++-----
.../cassandra/io/sstable/SSTableReader.java | 6 +-
.../apache/cassandra/service/CacheService.java | 2 +-
.../cassandra/service/CassandraDaemon.java | 9 +-
.../cassandra/tools/SSTableLevelResetter.java | 14 ++-
.../cassandra/cache/AutoSavingCacheTest.java | 66 ++++++++++++++
.../apache/cassandra/cache/ObjectSizeTest.java | 2 +-
.../cassandra/db/ColumnFamilyStoreTest.java | 8 +-
.../apache/cassandra/db/DirectoriesTest.java | 93 +++++++++++---------
.../io/sstable/SSTableSimpleWriterTest.java | 3 +-
19 files changed, 334 insertions(+), 125 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 45a3eb6..24722b8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -21,6 +21,8 @@
(CASSANDRA-5417, CASSANDRA-6520)
* Add snapshot space used to cfstats (CASSANDRA-6231)
* Add cardinality estimator for key count estimation (CASSANDRA-5906)
+ * CF id is changed to be non-deterministic. Data dir/key cache are created
+ uniquely for CF id (CASSANDRA-5202)
2.0.5
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index 14276d1..c7f8cb7 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -16,6 +16,16 @@ using the provided 'sstableupgrade' tool.
2.1
===
+New features
+------------
+ - SSTable data directory name is slightly changed. Each directory will
+ have hex string appended after CF name, e.g.
+ ks/cf-5be396077b811e3a3ab9dc4b9ac088d/
+ This hex string part represents unique ColumnFamily ID.
+ Note that existing directories are used as is, so only newly created
+ directories after upgrade have new directory name format.
+ - Saved key cache files also have ColumnFamily ID in their file name.
+
Upgrading
---------
- Rolling upgrades from anything pre-2.0 is not supported.
@@ -29,7 +39,6 @@ Upgrading
behavior) and 1.0 omits everything.
- Multithreaded compaction has been removed.
-
2.0.5
=====
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/src/java/org/apache/cassandra/cache/AutoSavingCache.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cache/AutoSavingCache.java b/src/java/org/apache/cassandra/cache/AutoSavingCache.java
index cbadefc..3ed2c2c 100644
--- a/src/java/org/apache/cassandra/cache/AutoSavingCache.java
+++ b/src/java/org/apache/cassandra/cache/AutoSavingCache.java
@@ -63,9 +63,9 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
this.cacheLoader = cacheloader;
}
- public File getCachePath(String ksName, String cfName, String version)
+ public File getCachePath(String ksName, String cfName, UUID cfId, String version)
{
- return DatabaseDescriptor.getSerializedCachePath(ksName, cfName, cacheType, version);
+ return DatabaseDescriptor.getSerializedCachePath(ksName, cfName, cfId, cacheType, version);
}
public Writer getWriter(int keysToSave)
@@ -102,7 +102,10 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
long start = System.nanoTime();
// modern format, allows both key and value (so key cache load can be purely sequential)
- File path = getCachePath(cfs.keyspace.getName(), cfs.name, CURRENT_VERSION);
+ File path = getCachePath(cfs.keyspace.getName(), cfs.name, cfs.metadata.cfId, CURRENT_VERSION);
+ // if path does not exist, try without cfId (assuming saved cache is created with current CF)
+ if (!path.exists())
+ path = getCachePath(cfs.keyspace.getName(), cfs.name, null, CURRENT_VERSION);
if (path.exists())
{
DataInputStream in = null;
@@ -199,13 +202,13 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
long start = System.nanoTime();
- HashMap<Pair<String, String>, SequentialWriter> writers = new HashMap<Pair<String, String>, SequentialWriter>();
+ HashMap<CacheKey.PathInfo, SequentialWriter> writers = new HashMap<>();
try
{
for (K key : keys)
{
- Pair<String, String> path = key.getPathInfo();
+ CacheKey.PathInfo path = key.getPathInfo();
SequentialWriter writer = writers.get(path);
if (writer == null)
{
@@ -231,13 +234,13 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
FileUtils.closeQuietly(writer);
}
- for (Map.Entry<Pair<String, String>, SequentialWriter> info : writers.entrySet())
+ for (Map.Entry<CacheKey.PathInfo, SequentialWriter> info : writers.entrySet())
{
- Pair<String, String> path = info.getKey();
+ CacheKey.PathInfo path = info.getKey();
SequentialWriter writer = info.getValue();
File tmpFile = new File(writer.getPath());
- File cacheFile = getCachePath(path.left, path.right, CURRENT_VERSION);
+ File cacheFile = getCachePath(path.keyspace, path.columnFamily, path.cfId, CURRENT_VERSION);
cacheFile.delete(); // ignore error if it didn't exist
if (!tmpFile.renameTo(cacheFile))
@@ -247,9 +250,9 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
logger.info("Saved {} ({} items) in {} ms", cacheType, keys.size(), TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start));
}
- private SequentialWriter tempCacheFile(Pair<String, String> pathInfo)
+ private SequentialWriter tempCacheFile(CacheKey.PathInfo pathInfo)
{
- File path = getCachePath(pathInfo.left, pathInfo.right, CURRENT_VERSION);
+ File path = getCachePath(pathInfo.keyspace, pathInfo.columnFamily, pathInfo.cfId, CURRENT_VERSION);
File tmpFile = FileUtils.createTempFile(path.getName(), null, path.getParentFile());
return SequentialWriter.open(tmpFile, true);
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/src/java/org/apache/cassandra/cache/CacheKey.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cache/CacheKey.java b/src/java/org/apache/cassandra/cache/CacheKey.java
index aa9f5f6..b4e6c22 100644
--- a/src/java/org/apache/cassandra/cache/CacheKey.java
+++ b/src/java/org/apache/cassandra/cache/CacheKey.java
@@ -17,12 +17,46 @@
*/
package org.apache.cassandra.cache;
-import org.apache.cassandra.utils.Pair;
+import java.util.UUID;
public interface CacheKey extends IMeasurableMemory
{
/**
* @return The keyspace and ColumnFamily names to which this key belongs
*/
- public Pair<String, String> getPathInfo();
+ public PathInfo getPathInfo();
+
+ public static class PathInfo
+ {
+ public final String keyspace;
+ public final String columnFamily;
+ public final UUID cfId;
+
+ public PathInfo(String keyspace, String columnFamily, UUID cfId)
+ {
+ this.keyspace = keyspace;
+ this.columnFamily = columnFamily;
+ this.cfId = cfId;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ PathInfo pathInfo = (PathInfo) o;
+
+ return (cfId != null ? cfId.equals(pathInfo.cfId) : pathInfo.cfId == null) && columnFamily.equals(pathInfo.columnFamily) && keyspace.equals(pathInfo.keyspace);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ int result = keyspace.hashCode();
+ result = 31 * result + columnFamily.hashCode();
+ result = 31 * result + (cfId != null ? cfId.hashCode() : 0);
+ return result;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/src/java/org/apache/cassandra/cache/KeyCacheKey.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cache/KeyCacheKey.java b/src/java/org/apache/cassandra/cache/KeyCacheKey.java
index 3b2077c..1133d94 100644
--- a/src/java/org/apache/cassandra/cache/KeyCacheKey.java
+++ b/src/java/org/apache/cassandra/cache/KeyCacheKey.java
@@ -19,30 +19,32 @@ package org.apache.cassandra.cache;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.UUID;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.ObjectSizes;
-import org.apache.cassandra.utils.Pair;
public class KeyCacheKey implements CacheKey
{
+ public final UUID cfId;
public final Descriptor desc;
// keeping an array instead of a ByteBuffer lowers the overhead of the key cache working set,
// without extra copies on lookup since client-provided key ByteBuffers will be array-backed already
public final byte[] key;
- public KeyCacheKey(Descriptor desc, ByteBuffer key)
+ public KeyCacheKey(UUID cfId, Descriptor desc, ByteBuffer key)
{
+ this.cfId = cfId;
this.desc = desc;
this.key = ByteBufferUtil.getArray(key);
assert this.key != null;
}
- public Pair<String, String> getPathInfo()
+ public PathInfo getPathInfo()
{
- return Pair.create(desc.ksname, desc.cfname);
+ return new PathInfo(desc.ksname, desc.cfname, cfId);
}
public String toString()
@@ -52,7 +54,9 @@ public class KeyCacheKey implements CacheKey
public long memorySize()
{
- return ObjectSizes.getFieldSize(// desc
+ return ObjectSizes.getFieldSize(// cfId
+ ObjectSizes.getReferenceSize() +
+ // desc
ObjectSizes.getReferenceSize() +
// key
ObjectSizes.getReferenceSize())
@@ -67,15 +71,15 @@ public class KeyCacheKey implements CacheKey
KeyCacheKey that = (KeyCacheKey) o;
- if (desc != null ? !desc.equals(that.desc) : that.desc != null) return false;
- return Arrays.equals(key, that.key);
+ return cfId.equals(that.cfId) && desc.equals(that.desc) && Arrays.equals(key, that.key);
}
@Override
public int hashCode()
{
- int result = desc != null ? desc.hashCode() : 0;
- result = 31 * result + (key != null ? Arrays.hashCode(key) : 0);
+ int result = cfId.hashCode();
+ result = 31 * result + desc.hashCode();
+ result = 31 * result + Arrays.hashCode(key);
return result;
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/src/java/org/apache/cassandra/cache/RowCacheKey.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cache/RowCacheKey.java b/src/java/org/apache/cassandra/cache/RowCacheKey.java
index 33e2065..fc4a6e0 100644
--- a/src/java/org/apache/cassandra/cache/RowCacheKey.java
+++ b/src/java/org/apache/cassandra/cache/RowCacheKey.java
@@ -45,9 +45,10 @@ public class RowCacheKey implements CacheKey, Comparable<RowCacheKey>
assert this.key != null;
}
- public Pair<String, String> getPathInfo()
+ public PathInfo getPathInfo()
{
- return Schema.instance.getCF(cfId);
+ Pair<String, String> cf = Schema.instance.getCF(cfId);
+ return new PathInfo(cf.left, cf.right, cfId);
}
public long memorySize()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/src/java/org/apache/cassandra/config/CFMetaData.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/CFMetaData.java b/src/java/org/apache/cassandra/config/CFMetaData.java
index ddc839c..d8ae26a 100644
--- a/src/java/org/apache/cassandra/config/CFMetaData.java
+++ b/src/java/org/apache/cassandra/config/CFMetaData.java
@@ -59,6 +59,7 @@ import org.apache.cassandra.thrift.CqlResult;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.UUIDGen;
import static org.apache.cassandra.utils.FBUtilities.*;
@@ -119,6 +120,7 @@ public final class CFMetaData
public static final CFMetaData SchemaColumnFamiliesCf = compile("CREATE TABLE " + SystemKeyspace.SCHEMA_COLUMNFAMILIES_CF + " ("
+ "keyspace_name text,"
+ "columnfamily_name text,"
+ + "cf_id uuid," // post-2.1 UUID cfid
+ "type text,"
+ "comparator text,"
+ "subcomparator text,"
@@ -452,9 +454,17 @@ public final class CFMetaData
public CFMetaData droppedColumns(Map<ColumnIdentifier, Long> cols) {droppedColumns = cols; return this;}
public CFMetaData triggers(Map<String, TriggerDefinition> prop) {triggers = prop; return this;}
+ /**
+ * Create new ColumnFamily metadata with generated random ID.
+ * When loading from existing schema, use CFMetaData
+ *
+ * @param keyspace keyspace name
+ * @param name column family name
+ * @param comp default comparator
+ */
public CFMetaData(String keyspace, String name, ColumnFamilyType type, CellNameType comp)
{
- this(keyspace, name, type, comp, getId(keyspace, name));
+ this(keyspace, name, type, comp, UUIDGen.getTimeUUID());
}
@VisibleForTesting
@@ -523,6 +533,12 @@ public final class CFMetaData
return (comment == null) ? "" : comment.toString();
}
+ /**
+ * Generates deterministic UUID from keyspace/columnfamily name pair.
+ * This is used to generate the same UUID for C* version < 2.1
+ *
+ * Since 2.1, this is only used for system columnfamilies and tests.
+ */
static UUID getId(String ksName, String cfName)
{
return UUID.nameUUIDFromBytes(ArrayUtils.addAll(ksName.getBytes(), cfName.getBytes()));
@@ -530,7 +546,7 @@ public final class CFMetaData
private static CFMetaData newSystemMetadata(String keyspace, String cfName, String comment, CellNameType comparator)
{
- CFMetaData newCFMD = new CFMetaData(keyspace, cfName, ColumnFamilyType.Standard, comparator);
+ CFMetaData newCFMD = new CFMetaData(keyspace, cfName, ColumnFamilyType.Standard, comparator, getId(keyspace, cfName));
return newCFMD.comment(comment)
.readRepairChance(0)
.dcLocalReadRepairChance(0)
@@ -967,10 +983,15 @@ public final class CFMetaData
boolean isDense = (cf_def.column_metadata == null || cf_def.column_metadata.isEmpty()) && !isCQL3OnlyPKComparator(rawComparator);
CellNameType comparator = CellNames.fromAbstractType(makeRawAbstractType(rawComparator, subComparator), isDense);
+ UUID cfId = Schema.instance.getId(cf_def.keyspace, cf_def.name);
+ if (cfId == null)
+ cfId = UUIDGen.getTimeUUID();
+
CFMetaData newCFMD = new CFMetaData(cf_def.keyspace,
cf_def.name,
cfType,
- comparator);
+ comparator,
+ cfId);
if (cf_def.isSetGc_grace_seconds()) { newCFMD.gcGraceSeconds(cf_def.gc_grace_seconds); }
if (cf_def.isSetMin_compaction_threshold()) { newCFMD.minCompactionThreshold(cf_def.min_compaction_threshold); }
@@ -1022,7 +1043,7 @@ public final class CFMetaData
/**
* Create CFMetaData from thrift {@link CqlRow} that contains columns from schema_columnfamilies.
*
- * @param cf CqlRow containing columns from schema_columnfamilies.
+ * @param columnsRes CqlRow containing columns from schema_columnfamilies.
* @return CFMetaData derived from CqlRow
*/
public static CFMetaData fromThriftCqlRow(CqlRow cf, CqlResult columnsRes)
@@ -1570,6 +1591,7 @@ public final class CFMetaData
Composite prefix = SchemaColumnFamiliesCf.comparator.make(cfName);
CFRowAdder adder = new CFRowAdder(cf, prefix, timestamp);
+ adder.add("cf_id", cfId);
adder.add("type", cfType.toString());
if (isSuper())
@@ -1637,10 +1659,18 @@ public final class CFMetaData
CellNameType comparator = CellNames.fromAbstractType(fullRawComparator, isDense(fullRawComparator, columnDefs));
+ // if we are upgrading, we use id generated from names initially
+ UUID cfId;
+ if (result.has("cf_id"))
+ cfId = result.getUUID("cf_id");
+ else
+ cfId = getId(ksName, cfName);
+
CFMetaData cfm = new CFMetaData(ksName,
cfName,
cfType,
- comparator);
+ comparator,
+ cfId);
cfm.readRepairChance(result.getDouble("read_repair_chance"));
cfm.dcLocalReadRepairChance(result.getDouble("local_read_repair_chance"));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 89c0228..6b49d21 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -39,7 +39,6 @@ import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.FSWriteError;
-import org.apache.cassandra.io.sstable.IndexSummaryManager;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.util.IAllocator;
import org.apache.cassandra.locator.DynamicEndpointSnitch;
@@ -51,6 +50,7 @@ import org.apache.cassandra.scheduler.IRequestScheduler;
import org.apache.cassandra.scheduler.NoScheduler;
import org.apache.cassandra.service.CacheService;
import org.apache.cassandra.utils.Allocator;
+import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
public class DatabaseDescriptor
@@ -1133,9 +1133,16 @@ public class DatabaseDescriptor
return conf.index_interval;
}
- public static File getSerializedCachePath(String ksName, String cfName, CacheService.CacheType cacheType, String version)
+ public static File getSerializedCachePath(String ksName, String cfName, UUID cfId, CacheService.CacheType cacheType, String version)
{
- return new File(conf.saved_caches_directory, ksName + "-" + cfName + "-" + cacheType + (version == null ? "" : "-" + version + ".db"));
+ StringBuilder builder = new StringBuilder();
+ builder.append(ksName).append('-');
+ builder.append(cfName).append('-');
+ if (cfId != null)
+ builder.append(ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(cfId))).append('-');
+ builder.append(cacheType);
+ builder.append((version == null ? "" : "-" + version + ".db"));
+ return new File(conf.saved_caches_directory, builder.toString());
}
public static int getDynamicUpdateInterval()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 2389d97..6d3e21a 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -397,7 +397,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
boolean loadSSTables)
{
// get the max generation number, to prevent generation conflicts
- Directories directories = Directories.create(keyspace.getName(), columnFamily);
+ Directories directories = new Directories(metadata);
Directories.SSTableLister lister = directories.sstableLister().includeBackups(true);
List<Integer> generations = new ArrayList<Integer>();
for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
@@ -417,11 +417,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
* Removes unnecessary files from the cf directory at startup: these include temp files, orphans, zero-length files
* and compacted sstables. Files that cannot be recognized will be ignored.
*/
- public static void scrubDataDirectories(String keyspaceName, String columnFamily)
+ public static void scrubDataDirectories(CFMetaData metadata)
{
- logger.debug("Removing compacted SSTable files from {} (see http://wiki.apache.org/cassandra/MemtableSSTable)", columnFamily);
+ logger.debug("Removing compacted SSTable files from {} (see http://wiki.apache.org/cassandra/MemtableSSTable)", metadata.cfName);
- Directories directories = Directories.create(keyspaceName, columnFamily);
+ Directories directories = new Directories(metadata);
for (Map.Entry<Descriptor,Set<Component>> sstableFiles : directories.sstableLister().list().entrySet())
{
Descriptor desc = sstableFiles.getKey();
@@ -447,7 +447,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
// cleanup incomplete saved caches
- Pattern tmpCacheFilePattern = Pattern.compile(keyspaceName + "-" + columnFamily + "-(Key|Row)Cache.*\\.tmp$");
+ Pattern tmpCacheFilePattern = Pattern.compile(metadata.ksName + "-" + metadata.cfName + "-(Key|Row)Cache.*\\.tmp$");
File dir = new File(DatabaseDescriptor.getSavedCachesLocation());
if (dir.exists())
@@ -460,11 +460,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
}
// also clean out any index leftovers.
- CFMetaData cfm = Schema.instance.getCFMetaData(keyspaceName, columnFamily);
- if (cfm != null) // secondary indexes aren't stored in DD.
+ for (ColumnDefinition def : metadata.allColumns())
{
- for (ColumnDefinition def : cfm.allColumns())
- scrubDataDirectories(keyspaceName, cfm.indexColumnFamilyName(def));
+ if (def.isIndexed())
+ {
+ CFMetaData indexMetadata = CFMetaData.newIndexMetadata(metadata, def, SecondaryIndex.getIndexComparator(metadata, def));
+ scrubDataDirectories(indexMetadata);
+ }
}
}
@@ -480,9 +482,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
* compactions, we remove the new ones (since those may be incomplete -- under LCS, we may create multiple
* sstables from any given ancestor).
*/
- public static void removeUnfinishedCompactionLeftovers(String keyspace, String columnfamily, Map<Integer, UUID> unfinishedCompactions)
+ public static void removeUnfinishedCompactionLeftovers(CFMetaData metadata, Map<Integer, UUID> unfinishedCompactions)
{
- Directories directories = Directories.create(keyspace, columnfamily);
+ Directories directories = new Directories(metadata);
Set<Integer> allGenerations = new HashSet<>();
for (Descriptor desc : directories.sstableLister().list().keySet())
@@ -495,7 +497,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
HashSet<Integer> missingGenerations = new HashSet<>(unfinishedGenerations);
missingGenerations.removeAll(allGenerations);
logger.debug("Unfinished compactions of {}.{} reference missing sstables of generations {}",
- keyspace, columnfamily, missingGenerations);
+ metadata.ksName, metadata.cfName, missingGenerations);
}
// remove new sstables from compactions that didn't complete, and compute
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 6f00d6d..9eb254e 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -33,9 +33,12 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSet.Builder;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
import com.google.common.primitives.Longs;
import com.google.common.util.concurrent.Uninterruptibles;
@@ -49,16 +52,21 @@ import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.sstable.*;
import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
/**
* Encapsulate handling of paths to the data files.
*
- * The directory layout is the following:
- * /<path_to_data_dir>/ks/cf1/ks-cf1-hb-1-Data.db
- * /cf2/ks-cf2-hb-1-Data.db
+ * Since v2.1, the directory layout is the following:
+ * /<path_to_data_dir>/ks/cf1-cfId/ks-cf1-ka-1-Data.db
+ * /cf2-cfId/ks-cf2-ka-1-Data.db
* ...
*
+ * cfId is an hex encoded CFID.
+ *
+ * For backward compatibility, Directories uses older directory layout if exists.
+ *
* In addition, more that one 'root' data directory can be specified so that
* <path_to_data_dir> potentially represents multiple locations.
* Note that in the case of multiple locations, the manifest for the leveled
@@ -162,27 +170,49 @@ public class Directories
}
}
- private final String keyspacename;
- private final String cfname;
+ private final CFMetaData metadata;
private final File[] sstableDirectories;
- public static Directories create(String keyspacename, String cfname)
+ /**
+ * Create Directories of given ColumnFamily.
+ * SSTable directories are created under data_directories defined in cassandra.yaml if not exist at this time.
+ *
+ * @param metadata metadata of ColumnFamily
+ */
+ public Directories(CFMetaData metadata)
{
- int idx = cfname.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
+ this.metadata = metadata;
+ this.sstableDirectories = new File[dataFileLocations.length];
+
+ // Determine SSTable directories
+ // If upgraded from version less than 2.1, use directories already exist.
+ for (int i = 0; i < dataFileLocations.length; ++i)
+ {
+ // check if old SSTable directory exists
+ sstableDirectories[i] = new File(dataFileLocations[i].location, join(metadata.ksName, metadata.cfName));
+ }
+ boolean olderDirectoryExists = Iterables.any(Arrays.asList(sstableDirectories), new Predicate<File>()
+ {
+ public boolean apply(File file)
+ {
+ return file.exists();
+ }
+ });
+ if (olderDirectoryExists)
+ return;
+
+ // create directory name
+ String directoryName;
+ String cfId = ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(metadata.cfId));
+ int idx = metadata.cfName.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
if (idx > 0)
// secondary index, goes in the same directory than the base cf
- return new Directories(keyspacename, cfname, cfname.substring(0, idx));
+ directoryName = metadata.cfName.substring(0, idx) + "-" + cfId;
else
- return new Directories(keyspacename, cfname, cfname);
- }
+ directoryName = metadata.cfName + "-" + cfId;
- private Directories(String keyspacename, String cfname, String directoryName)
- {
- this.keyspacename = keyspacename;
- this.cfname = cfname;
- this.sstableDirectories = new File[dataFileLocations.length];
for (int i = 0; i < dataFileLocations.length; ++i)
- sstableDirectories[i] = new File(dataFileLocations[i].location, join(keyspacename, directoryName));
+ sstableDirectories[i] = new File(dataFileLocations[i].location, join(metadata.ksName, directoryName));
if (!StorageService.instance.isClientMode())
{
@@ -251,7 +281,7 @@ public class Directories
*/
public DataDirectory getWriteableLocation()
{
- List<DataDirectory> candidates = new ArrayList<DataDirectory>();
+ List<DataDirectory> candidates = new ArrayList<>();
// pick directories with enough space and so that resulting sstable dirs aren't blacklisted for writes.
for (DataDirectory dataDir : dataFileLocations)
@@ -329,7 +359,7 @@ public class Directories
private boolean includeBackups;
private boolean onlyBackups;
private int nbFiles;
- private final Map<Descriptor, Set<Component>> components = new HashMap<Descriptor, Set<Component>>();
+ private final Map<Descriptor, Set<Component>> components = new HashMap<>();
private boolean filtered;
private String snapshotName;
@@ -375,7 +405,7 @@ public class Directories
public List<File> listFiles()
{
filter();
- List<File> l = new ArrayList<File>(nbFiles);
+ List<File> l = new ArrayList<>(nbFiles);
for (Map.Entry<Descriptor, Set<Component>> entry : components.entrySet())
{
for (Component c : entry.getValue())
@@ -434,7 +464,7 @@ public class Directories
Set<Component> previous = components.get(pair.left);
if (previous == null)
{
- previous = new HashSet<Component>();
+ previous = new HashSet<>();
components.put(pair.left, previous);
}
previous.add(pair.right);
@@ -494,7 +524,7 @@ public class Directories
private String getSSTablePrefix()
{
- return keyspacename + Component.separator + cfname + Component.separator;
+ return metadata.ksName + Component.separator + metadata.cfName + Component.separator;
}
public long getTrueAllocatedSizeIn(File input)
@@ -518,7 +548,7 @@ public class Directories
// Recursively finds all the sub directories in the KS directory.
public static List<File> getKSChildDirectories(String ksName)
{
- List<File> result = new ArrayList<File>();
+ List<File> result = new ArrayList<>();
for (DataDirectory dataDirectory : dataFileLocations)
{
File ksDir = new File(dataDirectory.location, ksName);
@@ -536,7 +566,7 @@ public class Directories
public List<File> getCFDirectories()
{
- List<File> result = new ArrayList<File>();
+ List<File> result = new ArrayList<>();
for (File dataDirectory : sstableDirectories)
{
if (dataDirectory.isDirectory())
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
index 30bfd77..53c315d 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@ -1058,7 +1058,7 @@ public class SSTableReader extends SSTable implements Closeable
return;
}
- KeyCacheKey cacheKey = new KeyCacheKey(descriptor, key.key);
+ KeyCacheKey cacheKey = new KeyCacheKey(metadata.cfId, descriptor, key.key);
logger.trace("Adding cache entry for {} -> {}", cacheKey, info);
keyCache.put(cacheKey, info);
}
@@ -1088,7 +1088,7 @@ public class SSTableReader extends SSTable implements Closeable
public RowIndexEntry getCachedPosition(DecoratedKey key, boolean updateStats)
{
- return getCachedPosition(new KeyCacheKey(descriptor, key.key), updateStats);
+ return getCachedPosition(new KeyCacheKey(metadata.cfId, descriptor, key.key), updateStats);
}
private RowIndexEntry getCachedPosition(KeyCacheKey unifiedKey, boolean updateStats)
@@ -1143,7 +1143,7 @@ public class SSTableReader extends SSTable implements Closeable
if ((op == Operator.EQ || op == Operator.GE) && (key instanceof DecoratedKey))
{
DecoratedKey decoratedKey = (DecoratedKey)key;
- KeyCacheKey cacheKey = new KeyCacheKey(descriptor, decoratedKey.key);
+ KeyCacheKey cacheKey = new KeyCacheKey(metadata.cfId, descriptor, decoratedKey.key);
RowIndexEntry cachedPosition = getCachedPosition(cacheKey, updateCacheAndStats);
if (cachedPosition != null)
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/src/java/org/apache/cassandra/service/CacheService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/CacheService.java b/src/java/org/apache/cassandra/service/CacheService.java
index 4fe7109..db024de 100644
--- a/src/java/org/apache/cassandra/service/CacheService.java
+++ b/src/java/org/apache/cassandra/service/CacheService.java
@@ -366,7 +366,7 @@ public class CacheService implements CacheServiceMBean
return null;
}
RowIndexEntry entry = reader.metadata.comparator.rowIndexEntrySerializer().deserialize(input, reader.descriptor.version);
- return Futures.immediateFuture(Pair.create(new KeyCacheKey(reader.descriptor, key), entry));
+ return Futures.immediateFuture(Pair.create(new KeyCacheKey(cfs.metadata.cfId, reader.descriptor, key), entry));
}
private SSTableReader findDesc(int generation, Collection<SSTableReader> collection)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/src/java/org/apache/cassandra/service/CassandraDaemon.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java
index 02731d8..ccabad5 100644
--- a/src/java/org/apache/cassandra/service/CassandraDaemon.java
+++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java
@@ -186,7 +186,7 @@ public class CassandraDaemon
// we do a one-off scrub of the system keyspace first; we can't load the list of the rest of the keyspaces,
// until system keyspace is opened.
for (CFMetaData cfm : Schema.instance.getKeyspaceMetaData(Keyspace.SYSTEM_KS).values())
- ColumnFamilyStore.scrubDataDirectories(Keyspace.SYSTEM_KS, cfm.cfName);
+ ColumnFamilyStore.scrubDataDirectories(cfm);
try
{
SystemKeyspace.checkHealth();
@@ -203,14 +203,17 @@ public class CassandraDaemon
// clean up compaction leftovers
Map<Pair<String, String>, Map<Integer, UUID>> unfinishedCompactions = SystemKeyspace.getUnfinishedCompactions();
for (Pair<String, String> kscf : unfinishedCompactions.keySet())
- ColumnFamilyStore.removeUnfinishedCompactionLeftovers(kscf.left, kscf.right, unfinishedCompactions.get(kscf));
+ {
+ CFMetaData cfm = Schema.instance.getCFMetaData(kscf.left, kscf.right);
+ ColumnFamilyStore.removeUnfinishedCompactionLeftovers(cfm, unfinishedCompactions.get(kscf));
+ }
SystemKeyspace.discardCompactionsInProgress();
// clean up debris in the rest of the keyspaces
for (String keyspaceName : Schema.instance.getKeyspaces())
{
for (CFMetaData cfm : Schema.instance.getKeyspaceMetaData(keyspaceName).values())
- ColumnFamilyStore.scrubDataDirectories(keyspaceName, cfm.cfName);
+ ColumnFamilyStore.scrubDataDirectories(cfm);
}
// initialize keyspaces
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
index aa555fc..2d95ad2 100644
--- a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
+++ b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
@@ -22,7 +22,9 @@ import java.io.PrintStream;
import java.util.Map;
import java.util.Set;
-import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
@@ -54,11 +56,15 @@ public class SSTableLevelResetter
System.exit(1);
}
- String keyspace = args[1];
+ // load keyspace descriptions.
+ DatabaseDescriptor.loadSchemas();
+
+ String keyspaceName = args[1];
String columnfamily = args[2];
- Directories directories = Directories.create(keyspace, columnfamily);
+ Keyspace keyspace = Keyspace.openWithoutSSTables(keyspaceName);
+ ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnfamily);
boolean foundSSTable = false;
- for (Map.Entry<Descriptor, Set<Component>> sstable : directories.sstableLister().list().entrySet())
+ for (Map.Entry<Descriptor, Set<Component>> sstable : cfs.directories.sstableLister().list().entrySet())
{
if (sstable.getValue().contains(Component.STATS))
{
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
new file mode 100644
index 0000000..ef04746
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cache/AutoSavingCacheTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.cache;
+
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.RowIndexEntry;
+import org.apache.cassandra.db.Mutation;
+import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public class AutoSavingCacheTest extends SchemaLoader
+{
+ @Test
+ public void testSerializeAndLoadKeyCache() throws Exception
+ {
+ ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
+ for (int i = 0; i < 2; i++)
+ {
+ Mutation rm = new Mutation("Keyspace1", ByteBufferUtil.bytes("key1"));
+ rm.add("Standard1", Util.cellname("c1"), ByteBufferUtil.bytes(i), 0);
+ rm.apply();
+ cfs.forceBlockingFlush();
+ }
+
+ assert cfs.getSSTables().size() == 2;
+
+ // preheat key cache
+ for (SSTableReader sstable : cfs.getSSTables())
+ sstable.getPosition(Util.dk("key1"), SSTableReader.Operator.EQ);
+
+ AutoSavingCache<KeyCacheKey, RowIndexEntry> keyCache = CacheService.instance.keyCache;
+
+ // serialize to file
+ keyCache.submitWrite(keyCache.size()).get();
+ keyCache.clear();
+
+ assert keyCache.size() == 0;
+
+ // then load saved
+ keyCache.loadSaved(cfs);
+ assert keyCache.size() == 2;
+ for (SSTableReader sstable : cfs.getSSTables())
+ assert keyCache.get(new KeyCacheKey(cfs.metadata.cfId, sstable.descriptor, ByteBufferUtil.bytes("key1"))) != null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/test/unit/org/apache/cassandra/cache/ObjectSizeTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cache/ObjectSizeTest.java b/test/unit/org/apache/cassandra/cache/ObjectSizeTest.java
index 4bb9b20..da34711 100644
--- a/test/unit/org/apache/cassandra/cache/ObjectSizeTest.java
+++ b/test/unit/org/apache/cassandra/cache/ObjectSizeTest.java
@@ -60,7 +60,7 @@ public class ObjectSizeTest
@Test
public void testKeyCacheKey()
{
- KeyCacheKey key = new KeyCacheKey(null, ByteBuffer.wrap(new byte[0]));
+ KeyCacheKey key = new KeyCacheKey(null, null, ByteBuffer.wrap(new byte[0]));
long size = key.memorySize();
long size2 = meter.measureDeep(key);
Assert.assertEquals(size, size2);
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index 0e50cb6..5ca468f 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@ -1572,7 +1572,7 @@ public class ColumnFamilyStoreTest extends SchemaLoader
String cf = "Standard3"; // should be empty
final CFMetaData cfmeta = Schema.instance.getCFMetaData(ks, cf);
- Directories dir = Directories.create(ks, cf);
+ Directories dir = new Directories(cfmeta);
ByteBuffer key = bytes("key");
// 1st sstable
@@ -1616,7 +1616,7 @@ public class ColumnFamilyStoreTest extends SchemaLoader
Map<Integer, UUID> unfinishedCompaction = new HashMap<>();
unfinishedCompaction.put(sstable1.descriptor.generation, compactionTaskID);
- ColumnFamilyStore.removeUnfinishedCompactionLeftovers(ks, cf, unfinishedCompaction);
+ ColumnFamilyStore.removeUnfinishedCompactionLeftovers(cfmeta, unfinishedCompaction);
// 2nd sstable should be removed (only 1st sstable exists in set of size 1)
sstables = dir.sstableLister().list();
@@ -1637,7 +1637,7 @@ public class ColumnFamilyStoreTest extends SchemaLoader
final String cf = "Standard4"; // should be empty
final CFMetaData cfmeta = Schema.instance.getCFMetaData(ks, cf);
- Directories dir = Directories.create(ks, cf);
+ Directories dir = new Directories(cfmeta);
ByteBuffer key = bytes("key");
// Write SSTable generation 3 that has ancestors 1 and 2
@@ -1673,7 +1673,7 @@ public class ColumnFamilyStoreTest extends SchemaLoader
UUID compactionTaskID = UUID.randomUUID();
for (Integer ancestor : ancestors)
unfinishedCompactions.put(ancestor, compactionTaskID);
- ColumnFamilyStore.removeUnfinishedCompactionLeftovers(ks, cf, unfinishedCompactions);
+ ColumnFamilyStore.removeUnfinishedCompactionLeftovers(cfmeta, unfinishedCompactions);
// SSTable should not be deleted
sstables = dir.sstableLister().list();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/test/unit/org/apache/cassandra/db/DirectoriesTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
index bd7e318..a9cd2a3 100644
--- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java
+++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
@@ -23,28 +23,38 @@ import java.util.*;
import java.util.concurrent.*;
import org.junit.AfterClass;
-import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.Config.DiskFailurePolicy;
import org.apache.cassandra.db.Directories.DataDirectory;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class DirectoriesTest
{
private static File tempDataDir;
- private static String KS = "ks";
- private static String[] CFS = new String[] { "cf1", "ks" };
+ private static final String KS = "ks";
+ private static final String[] CFS = new String[] { "cf1", "ks" };
+ private static final Set<CFMetaData> CFM = new HashSet<>(CFS.length);
private static Map<String, List<File>> files = new HashMap<String, List<File>>();
@BeforeClass
public static void beforeClass() throws IOException
{
+ for (String cf : CFS)
+ {
+ CFM.add(new CFMetaData(KS, cf, ColumnFamilyType.Standard, null));
+ }
+
tempDataDir = File.createTempFile("cassandra", "unittest");
tempDataDir.delete(); // hack to create a temp dir
tempDataDir.mkdir();
@@ -63,23 +73,23 @@ public class DirectoriesTest
private static void createTestFiles() throws IOException
{
- for (String cf : CFS)
+ for (CFMetaData cfm : CFM)
{
- List<File> fs = new ArrayList<File>();
- files.put(cf, fs);
- File dir = cfDir(cf);
+ List<File> fs = new ArrayList<>();
+ files.put(cfm.cfName, fs);
+ File dir = cfDir(cfm);
dir.mkdirs();
- createFakeSSTable(dir, cf, 1, false, fs);
- createFakeSSTable(dir, cf, 2, true, fs);
+ createFakeSSTable(dir, cfm.cfName, 1, false, fs);
+ createFakeSSTable(dir, cfm.cfName, 2, true, fs);
File backupDir = new File(dir, Directories.BACKUPS_SUBDIR);
backupDir.mkdir();
- createFakeSSTable(backupDir, cf, 1, false, fs);
+ createFakeSSTable(backupDir, cfm.cfName, 1, false, fs);
File snapshotDir = new File(dir, Directories.SNAPSHOT_SUBDIR + File.separator + "42");
snapshotDir.mkdirs();
- createFakeSSTable(snapshotDir, cf, 1, false, fs);
+ createFakeSSTable(snapshotDir, cfm.cfName, 1, false, fs);
}
}
@@ -94,41 +104,42 @@ public class DirectoriesTest
}
}
- private static File cfDir(String cf)
+ private static File cfDir(CFMetaData metadata)
{
- return new File(tempDataDir, KS + File.separator + cf);
+ String cfId = ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(metadata.cfId));
+ return new File(tempDataDir, metadata.ksName + File.separator + metadata.cfName + "-" + cfId);
}
@Test
public void testStandardDirs()
{
- for (String cf : CFS)
+ for (CFMetaData cfm : CFM)
{
- Directories directories = Directories.create(KS, cf);
- Assert.assertEquals(cfDir(cf), directories.getDirectoryForNewSSTables());
+ Directories directories = new Directories(cfm);
+ assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables());
- Descriptor desc = new Descriptor(cfDir(cf), KS, cf, 1, false);
- File snapshotDir = new File(cfDir(cf), File.separator + Directories.SNAPSHOT_SUBDIR + File.separator + "42");
- Assert.assertEquals(snapshotDir, directories.getSnapshotDirectory(desc, "42"));
+ Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.cfName, 1, false);
+ File snapshotDir = new File(cfDir(cfm), File.separator + Directories.SNAPSHOT_SUBDIR + File.separator + "42");
+ assertEquals(snapshotDir, Directories.getSnapshotDirectory(desc, "42"));
- File backupsDir = new File(cfDir(cf), File.separator + Directories.BACKUPS_SUBDIR);
- Assert.assertEquals(backupsDir, directories.getBackupsDirectory(desc));
+ File backupsDir = new File(cfDir(cfm), File.separator + Directories.BACKUPS_SUBDIR);
+ assertEquals(backupsDir, Directories.getBackupsDirectory(desc));
}
}
@Test
public void testSSTableLister()
{
- for (String cf : CFS)
+ for (CFMetaData cfm : CFM)
{
- Directories directories = Directories.create(KS, cf);
+ Directories directories = new Directories(cfm);
Directories.SSTableLister lister;
Set<File> listed;
// List all but no snapshot, backup
lister = directories.sstableLister();
- listed = new HashSet<File>(lister.listFiles());
- for (File f : files.get(cf))
+ listed = new HashSet<>(lister.listFiles());
+ for (File f : files.get(cfm.cfName))
{
if (f.getPath().contains(Directories.SNAPSHOT_SUBDIR) || f.getPath().contains(Directories.BACKUPS_SUBDIR))
assert !listed.contains(f) : f + " should not be listed";
@@ -138,8 +149,8 @@ public class DirectoriesTest
// List all but including backup (but no snapshot)
lister = directories.sstableLister().includeBackups(true);
- listed = new HashSet<File>(lister.listFiles());
- for (File f : files.get(cf))
+ listed = new HashSet<>(lister.listFiles());
+ for (File f : files.get(cfm.cfName))
{
if (f.getPath().contains(Directories.SNAPSHOT_SUBDIR))
assert !listed.contains(f) : f + " should not be listed";
@@ -149,8 +160,8 @@ public class DirectoriesTest
// Skip temporary and compacted
lister = directories.sstableLister().skipTemporary(true);
- listed = new HashSet<File>(lister.listFiles());
- for (File f : files.get(cf))
+ listed = new HashSet<>(lister.listFiles());
+ for (File f : files.get(cfm.cfName))
{
if (f.getPath().contains(Directories.SNAPSHOT_SUBDIR) || f.getPath().contains(Directories.BACKUPS_SUBDIR))
assert !listed.contains(f) : f + " should not be listed";
@@ -177,13 +188,13 @@ public class DirectoriesTest
dd.location.setExecutable(false);
dd.location.setWritable(false);
}
-
- Directories.create(KS, "bad");
-
- for (DataDirectory dd : Directories.dataFileLocations)
+
+ CFMetaData cfm = new CFMetaData(KS, "bad", ColumnFamilyType.Standard, null);
+ Directories dir = new Directories(cfm);
+
+ for (File file : dir.getCFDirectories())
{
- File file = new File(dd.location, new File(KS, "bad").getPath());
- Assert.assertTrue(BlacklistedDirectories.isUnwritable(file));
+ assertTrue(BlacklistedDirectories.isUnwritable(file));
}
}
finally
@@ -201,20 +212,20 @@ public class DirectoriesTest
@Test
public void testMTSnapshots() throws Exception
{
- for (final String cf : CFS)
+ for (final CFMetaData cfm : CFM)
{
- final Directories directories = Directories.create(KS, cf);
- Assert.assertEquals(cfDir(cf), directories.getDirectoryForNewSSTables());
+ final Directories directories = new Directories(cfm);
+ assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables());
final String n = Long.toString(System.nanoTime());
Callable<File> directoryGetter = new Callable<File>() {
public File call() throws Exception {
- Descriptor desc = new Descriptor(cfDir(cf), KS, cf, 1, false);
- return directories.getSnapshotDirectory(desc, n);
+ Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.cfName, 1, false);
+ return Directories.getSnapshotDirectory(desc, n);
}
};
List<Future<File>> invoked = Executors.newFixedThreadPool(2).invokeAll(Arrays.asList(directoryGetter, directoryGetter));
for(Future<File> fut:invoked) {
- Assert.assertTrue(fut.get().exists());
+ assertTrue(fut.get().exists());
}
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/0a1b277d/test/unit/org/apache/cassandra/io/sstable/SSTableSimpleWriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableSimpleWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableSimpleWriterTest.java
index 9aaeca5..15980a4 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableSimpleWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableSimpleWriterTest.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.io.sstable;
import java.io.File;
+import org.apache.cassandra.config.Schema;
import org.apache.cassandra.dht.IPartitioner;
import org.junit.Test;
@@ -43,7 +44,7 @@ public class SSTableSimpleWriterTest extends SchemaLoader
String cfname = "StandardInteger1";
Keyspace t = Keyspace.open(keyspaceName); // make sure we create the directory
- File dir = Directories.create(keyspaceName, cfname).getDirectoryForNewSSTables();
+ File dir = new Directories(Schema.instance.getCFMetaData(keyspaceName, cfname)).getDirectoryForNewSSTables();
assert dir.exists();
IPartitioner partitioner = StorageService.getPartitioner();