You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by pa...@apache.org on 2022/03/14 20:25:40 UTC
[cassandra] branch trunk updated (3b1ce69 -> 951645a)
This is an automated email from the ASF dual-hosted git repository.
paulo pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git.
from 3b1ce69 Merge branch 'cassandra-4.0' into trunk
new 95a6223 Fix snapshot true size calculation
new 5f50c79 Merge branch 'cassandra-3.11' into cassandra-4.0
new 951645a Merge branch 'cassandra-4.0' into trunk
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.txt | 1 +
src/java/org/apache/cassandra/db/Directories.java | 6 +--
.../apache/cassandra/db/ColumnFamilyStoreTest.java | 41 ++++++++++++++++++
.../apache/cassandra/index/sasi/SASIIndexTest.java | 48 +++++++++++++---------
4 files changed, 74 insertions(+), 22 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra] 01/01: Merge branch 'cassandra-4.0' into trunk
Posted by pa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
paulo pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 951645a1fc4486f2b941c4eb72c429780badca07
Merge: 3b1ce69 5f50c79
Author: Paulo Motta <pa...@apache.org>
AuthorDate: Mon Mar 14 17:21:51 2022 -0300
Merge branch 'cassandra-4.0' into trunk
CHANGES.txt | 1 +
src/java/org/apache/cassandra/db/Directories.java | 6 +--
.../apache/cassandra/db/ColumnFamilyStoreTest.java | 41 ++++++++++++++++++
.../apache/cassandra/index/sasi/SASIIndexTest.java | 48 +++++++++++++---------
4 files changed, 74 insertions(+), 22 deletions(-)
diff --cc CHANGES.txt
index c57742f,86eae2b..ea1826a
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -133,26 -46,15 +133,27 @@@ Merged from 4.0
* Avoid rewriting all sstables during cleanup when transient replication is enabled (CASSANDRA-16966)
* Prevent CQLSH from failure on Python 3.10 (CASSANDRA-16987)
* Avoid trying to acquire 0 permits from the rate limiter when taking snapshot (CASSANDRA-16872)
- * Upgrade Caffeine to 2.5.6 (CASSANDRA-15153)
- * Include SASI components to snapshots (CASSANDRA-15134)
- * Fix missed wait latencies in the output of `nodetool tpstats -F` (CASSANDRA-16938)
* Remove all the state pollution between tests in SSTableReaderTest (CASSANDRA-16888)
* Delay auth setup until after gossip has settled to avoid unavailables on startup (CASSANDRA-16783)
- * Fix clustering order logic in CREATE MATERIALIZED VIEW (CASSANDRA-16898)
* org.apache.cassandra.db.rows.ArrayCell#unsharedHeapSizeExcludingData includes data twice (CASSANDRA-16900)
+ * Fix clustering order logic in CREATE MATERIALIZED VIEW (CASSANDRA-16898)
* Exclude Jackson 1.x transitive dependency of hadoop* provided dependencies (CASSANDRA-16854)
+ * Tolerate missing DNS entry when completing a host replacement (CASSANDRA-16873)
+ * Harden PrunableArrayQueue against Pruner implementations that might throw exceptions (CASSANDRA-16866)
+ * Move RepairedDataInfo to the execution controller rather than the ReadCommand to avoid unintended sharing (CASSANDRA-16721)
+ * Bump zstd-jni version to 1.5.0-4 (CASSANDRA-16884)
+ * Remove assumption that all urgent messages are small (CASSANDRA-16877)
+ * ArrayClustering.unsharedHeapSize does not include the data so undercounts the heap size (CASSANDRA-16845)
+ * Improve help, doc and error messages about sstabledump -k and -x arguments (CASSANDRA-16818)
+ * Add repaired/unrepaired bytes back to nodetool (CASSANDRA-15282)
+ * Upgrade lz4-java to 1.8.0 to add RH6 support back (CASSANDRA-16753)
+ * Improve DiagnosticEventService.publish(event) logging message of events (CASSANDRA-16749)
+ * Cleanup dependency scopes (CASSANDRA-16704)
+ * Make JmxHistogram#getRecentValues() and JmxTimer#getRecentValues() thread-safe (CASSANDRA-16707)
Merged from 3.11:
++ * Fix snapshot true size calculation (CASSANDRA-17267)
+ * dropping of a materialized view creates a snapshot with dropped- prefix (CASSANDRA-17415)
+ * Validate existence of DCs when repairing (CASSANDRA-17407)
* Add key validation to ssstablescrub (CASSANDRA-16969)
* Update Jackson from 2.9.10 to 2.12.5 (CASSANDRA-16851)
* Make assassinate more resilient to missing tokens (CASSANDRA-16847)
diff --cc src/java/org/apache/cassandra/db/Directories.java
index 5a9c563,f09cdae..1cc350d
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@@ -1224,7 -1162,7 +1224,7 @@@ public class Directorie
SSTableSizeSummer(File path, List<File> files)
{
super(path);
- toSkip = new HashSet<>(files);
- toSkip = files.stream().map(f -> f.getName()).collect(Collectors.toSet());
++ toSkip = files.stream().map(f -> f.name()).collect(Collectors.toSet());
}
@Override
@@@ -1235,7 -1173,7 +1235,7 @@@
return desc != null
&& desc.ksname.equals(metadata.keyspace)
&& desc.cfname.equals(metadata.name)
- && !toSkip.contains(file);
- && !toSkip.contains(file.getName());
++ && !toSkip.contains(file.name());
}
}
diff --cc test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index e7c7e22,266b37d..d970b12
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@@ -29,10 -31,16 +29,11 @@@ import org.junit.Before
import org.junit.BeforeClass;
import org.junit.Test;
+ import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.utils.Pair;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.service.snapshot.SnapshotManifest;
+import org.apache.cassandra.service.snapshot.TableSnapshot;
import com.google.common.collect.Iterators;
import org.apache.cassandra.*;
@@@ -252,6 -258,46 +253,46 @@@ public class ColumnFamilyStoreTes
}
@Test
+ public void testSnapshotSize()
+ {
+ // cleanup any previous test gargbage
+ ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD1);
+ cfs.clearSnapshot("");
+
+ // Add row
+ new RowUpdateBuilder(cfs.metadata(), 0, "key1")
+ .clustering("Column1")
+ .add("val", "asdf")
+ .build()
+ .applyUnsafe();
+ cfs.forceBlockingFlush();
+
+ // snapshot
+ cfs.snapshot("basic", null, false, false);
+
+ // check snapshot was created
- Map<String, Directories.SnapshotSizeDetails> snapshotDetails = cfs.getSnapshotDetails();
++ Map<String, TableSnapshot> snapshotDetails = cfs.listSnapshots();
+ assertThat(snapshotDetails).hasSize(1);
+ assertThat(snapshotDetails).containsKey("basic");
+
+ // check that sizeOnDisk > trueSize = 0
- Directories.SnapshotSizeDetails details = snapshotDetails.get("basic");
- assertThat(details.sizeOnDiskBytes).isGreaterThan(details.dataSizeBytes);
- assertThat(details.dataSizeBytes).isZero();
++ TableSnapshot details = snapshotDetails.get("basic");
++ assertThat(details.computeSizeOnDiskBytes()).isGreaterThan(details.computeTrueSizeBytes());
++ assertThat(details.computeTrueSizeBytes()).isZero();
+
+ // compact base table to make trueSize > 0
+ cfs.forceMajorCompaction();
+ LifecycleTransaction.waitForDeletions();
+
+ // sizeOnDisk > trueSize because trueSize does not include manifest.json
+ // Check that truesize now is > 0
- snapshotDetails = cfs.getSnapshotDetails();
++ snapshotDetails = cfs.listSnapshots();
+ details = snapshotDetails.get("basic");
- assertThat(details.sizeOnDiskBytes).isGreaterThan(details.dataSizeBytes);
- assertThat(details.dataSizeBytes).isPositive();
++ assertThat(details.computeSizeOnDiskBytes()).isGreaterThan(details.computeTrueSizeBytes());
++ assertThat(details.computeTrueSizeBytes()).isPositive();
+ }
+
+ @Test
public void testBackupAfterFlush() throws Throwable
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE2).getColumnFamilyStore(CF_STANDARD1);
diff --cc test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
index 15dd5d8,945a5e9..10fff4b
--- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
@@@ -41,8 -42,8 +41,9 @@@ import org.apache.cassandra.cql3.CQLTes
import org.apache.cassandra.cql3.Operator;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.UntypedResultSet;
+ import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
@@@ -164,11 -165,17 +164,16 @@@ public class SASIIndexTes
try
{
store.snapshot(snapshotName);
++
+ // Compact to make true snapshot size != 0
+ store.forceMajorCompaction();
+ LifecycleTransaction.waitForDeletions();
+
- FileReader reader = new FileReader(store.getDirectories().getSnapshotManifestFile(snapshotName));
- JSONObject manifest = (JSONObject) new JSONParser().parse(reader);
- JSONArray files = (JSONArray) manifest.get("files");
+ SnapshotManifest manifest = SnapshotManifest.deserializeFromJsonFile(store.getDirectories().getSnapshotManifestFile(snapshotName));
Assert.assertFalse(ssTableReaders.isEmpty());
- Assert.assertFalse(files.isEmpty());
- Assert.assertEquals(ssTableReaders.size(), files.size());
+ Assert.assertFalse(manifest.files.isEmpty());
+ Assert.assertEquals(ssTableReaders.size(), manifest.files.size());
Map<Descriptor, Set<Component>> snapshotSSTables = store.getDirectories()
.sstableLister(Directories.OnTxnErr.IGNORE)
@@@ -202,11 -208,11 +206,11 @@@
tableSize += componentSize;
}
}
-
- Map<String, Directories.SnapshotSizeDetails> details = store.getSnapshotDetails();
+
+ TableSnapshot details = store.listSnapshots().get(snapshotName);
// check that SASI components are included in the computation of snapshot size
- Assert.assertEquals(details.computeTrueSizeBytes(), tableSize + indexSize);
- Assert.assertEquals(tableSize + indexSize, (long) details.get(snapshotName).dataSizeBytes);
++ Assert.assertEquals(tableSize + indexSize, details.computeTrueSizeBytes());
}
finally
{
@@@ -2651,21 -2661,25 +2659,24 @@@
do
{
count = 0;
- currentPage = getIndexed(store, ColumnFilter.all(store.metadata()), lastKey, pageSize, expressions);
- if (currentPage == null)
- break;
+ ReadCommand command = getIndexReadCommand(store, ColumnFilter.all(store.metadata()), lastKey, pageSize, expressions);
- while (currentPage.hasNext())
+ try (ReadExecutionController controller = command.executionController();
+ UnfilteredPartitionIterator currentPage = command.executeLocally(controller))
{
- try (UnfilteredRowIterator row = currentPage.next())
+ if (currentPage == null)
+ break;
+
+ while (currentPage.hasNext())
{
- uniqueKeys.add(row.partitionKey());
- lastKey = row.partitionKey();
- count++;
+ try (UnfilteredRowIterator row = currentPage.next())
+ {
+ uniqueKeys.add(row.partitionKey());
+ lastKey = row.partitionKey();
+ count++;
+ }
}
}
--
- currentPage.close();
}
while (count == pageSize);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org