You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ri...@apache.org on 2026/10/04 10:33:16 UTC
(asterixdb) branch master updated: [ASTERIXDB-3885][STO] Compress VTree data components
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new f35153a7f8 [ASTERIXDB-3885][STO] Compress VTree data components
f35153a7f8 is described below
commit f35153a7f81e98ebcfee7611a423f8737a034f86
Author: Ritik Raj <ra...@gmail.com>
AuthorDate: Fri Sep 25 12:42:34 2026 +0530
[ASTERIXDB-3885][STO] Compress VTree data components
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- The VTree never received a compression scheme: its file manager passed
null to the base, which resolves to NoOp, and built component
references without the look-aside file. The loaders already drive the
compressed page writer, so only the scheme and the file references
were missing.
- Like a B-tree secondary, a VTree now takes a scheme of its own,
recorded when it is created: the one its WITH clause names next to the
vector parameters, or the configured default.
CREATE INDEX i ON ds(emb VECTOR) TYPE VTREE
WITH {"dimension": 384, "similarity": "cosine",
"storage-block-compression": {"scheme": "none"}};
- Only .vct data components are compressed; the shared static structure
is written once, outside flush and merge, and stays uncompressed.
- Each page's unused tail is zeroed before it is written. A written page
is a reused buffer whose tail still held earlier data, and a VTree
leaves much of it unused since every cluster ends on a partly filled
page, so the tail compressed poorly and carried stale bytes.
- The scheme is persisted per index. A resource written before this has
no key and reads back as NoOp, so existing indexes keep opening.
- Recovery resolves components through the scheme, finishes a delete
that left a data file without its look-aside file (or the reverse),
and removes the look-aside files of merged-away inputs.
Generated-by: Claude Opus 5.5 <no...@anthropic.com>
Change-Id: I13f1ad75516025efb3a24a2d3715cdf519d27ae6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21887
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Tested-by: Ritik Raj <ri...@couchbase.com>
---
.../asterix/app/translator/QueryTranslator.java | 7 +-
.../queries_sqlpp/vector/VectorQueries.xml | 12 ++++
...eate-index-vtree-compression-scheme.1.ddl.sqlpp | 33 ++++++++++
...e-index-vtree-compression-scheme.2.update.sqlpp | 53 ++++++++++++++++
...eate-index-vtree-compression-scheme.3.ddl.sqlpp | 20 ++++++
...eate-index-vtree-compression-scheme.4.ddl.sqlpp | 31 +++++++++
...te-index-vtree-compression-scheme.5.query.sqlpp | 24 +++++++
...ate-index-vtree-compression-scheme.99.ddl.sqlpp | 20 ++++++
...ex-vtree-invalid-compression-scheme.1.ddl.sqlpp | 28 ++++++++
.../create-index-vtree-compression-scheme.5.adm | 3 +
.../create-index-vtree-metadata.5.adm | 2 +-
.../common/statement/CreateIndexStatement.java | 8 ++-
.../common/util/DatasetDeclParametersUtil.java | 20 ++++++
.../lang/common/util/VectorIndexDeclUtil.java | 10 ++-
.../utils/VTreeResourceFactoryProvider.java | 7 +-
.../lsm/vector/dataflow/LSMVTreeLocalResource.java | 30 +++++++--
.../dataflow/LSMVTreeLocalResourceFactory.java | 10 ++-
.../am/lsm/vector/impls/LSMVTreeFileManager.java | 43 +++++++++++--
.../storage/am/lsm/vector/utils/LSMVTreeUtils.java | 9 ++-
.../storage/am/vector/frames/VTreeNSMFrame.java | 23 +++++++
.../storage/am/vector/impls/VTreeBulkLoader.java | 2 +
.../storage/am/vector/impls/VTreeFlushLoader.java | 2 +
.../vector/impls/VTreeStaticStructureBuilder.java | 3 +
.../am/lsm/vector/LSMVTreeCompressedMergeTest.java | 53 ++++++++++++++++
.../LSMVTreeFileManagerCrashRecoveryTest.java | 74 +++++++++++++++++++++-
.../lsm/vector/LSMVTreeLocalResourceJsonTest.java | 53 ++++++++++++++--
.../storage/am/lsm/vector/LSMVTreeMergeTest.java | 24 ++++++-
.../am/lsm/vector/util/LSMVTreeTestContext.java | 23 ++++++-
28 files changed, 593 insertions(+), 34 deletions(-)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index e112bc36b9..e49ff63733 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -1924,6 +1924,11 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
}
+ // Resolved now, for the same reason as a BTREE's: the scheme its WITH clause names, or the
+ // configured default. Ahead of the sizing, so a bad scheme is reported before the dataset's state.
+ String compressionScheme = metadataProvider.getCompressionManager()
+ .getDdlOrDefaultCompressionScheme(stmtCreateIndex.getCompressionScheme());
+
// Fix the sizing before the record is written. num_clusters and train_list_fraction are
// derived from the per-partition cardinality when the user leaves them out, and the record is
// written before any build job runs, so deriving them later would leave the catalog silent
@@ -1934,7 +1939,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
includeFieldSourceIndicators, includeFieldTypes, false, vectorParameters);
Index newIndex = new Index(databaseName, dataverseName, datasetName, indexName, indexType, indexDetails,
- stmtCreateIndex.isEnforced(), false, MetadataUtil.PENDING_ADD_OP, creator);
+ stmtCreateIndex.isEnforced(), false, MetadataUtil.PENDING_ADD_OP, creator, compressionScheme);
bActiveTxn = false; // doCreateIndexImpl() takes over the current transaction
EntityDetails entityDetails =
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/VectorQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/VectorQueries.xml
index 918343a4a4..10141c5079 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/VectorQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/VectorQueries.xml
@@ -56,6 +56,18 @@
<output-dir compare="Clean-JSON">create-index-vtree-metadata</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="create-index-vtree-compression-scheme">
+ <output-dir compare="Clean-JSON">create-index-vtree-compression-scheme</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="vector">
+ <compilation-unit name="create-index-vtree-invalid-compression-scheme">
+ <output-dir compare="Text">create-index-vtree-invalid-compression-scheme</output-dir>
+ <expected-error>ASX1096: Unknown compression scheme zip. Supported schemes are [snappy,none]</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="vector">
<compilation-unit name="create-index-vtree-meta-pk">
<output-dir compare="Clean-JSON">create-index-vtree-meta-pk</output-dir>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.1.ddl.sqlpp
new file mode 100644
index 0000000000..dc2a169a17
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.1.ddl.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// A vector index takes its own block compression scheme, as a BTREE does: the one its WITH clause names, or the
+// configured default. The dataset is uncompressed, so an index that is compressed got it from its own clause.
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE MovieType AS {
+ id: int,
+ title: string,
+ info: { year: int },
+ embedding: [double]
+};
+
+CREATE DATASET MovieSmall(MovieType) PRIMARY KEY id WITH {"storage-block-compression": {"scheme": "none"}};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.2.update.sqlpp
new file mode 100644
index 0000000000..daa3061edc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.2.update.sqlpp
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+INSERT INTO MovieSmall ([
+ {"id": 1, "title": "t1", "info": {"year": 2001}, "embedding": [0.0, 0.0, 0.0, 0.1]},
+ {"id": 2, "title": "t2", "info": {"year": 2002}, "embedding": [0.0, 0.0, 0.0, 0.2]},
+ {"id": 3, "title": "t3", "info": {"year": 2003}, "embedding": [0.0, 0.0, 0.0, 0.3]},
+ {"id": 4, "title": "t4", "info": {"year": 2004}, "embedding": [0.0, 0.0, 0.0, 0.4]},
+ {"id": 5, "title": "t5", "info": {"year": 2005}, "embedding": [0.0, 0.0, 0.0, 0.5]},
+ {"id": 6, "title": "t6", "info": {"year": 2006}, "embedding": [0.0, 0.0, 0.0, 0.6]},
+ {"id": 7, "title": "t7", "info": {"year": 2007}, "embedding": [0.0, 0.0, 0.0, 0.7]},
+ {"id": 8, "title": "t8", "info": {"year": 2008}, "embedding": [0.0, 0.0, 0.0, 0.8]},
+ {"id": 9, "title": "t9", "info": {"year": 2009}, "embedding": [0.0, 0.0, 0.0, 0.9]},
+ {"id": 10, "title": "t10", "info": {"year": 2010}, "embedding": [0.0, 0.0, 0.0, 1.0]},
+ {"id": 11, "title": "t11", "info": {"year": 2011}, "embedding": [0.0, 0.0, 0.0, 1.1]},
+ {"id": 12, "title": "t12", "info": {"year": 2012}, "embedding": [0.0, 0.0, 0.0, 1.2]},
+ {"id": 13, "title": "t13", "info": {"year": 2013}, "embedding": [0.0, 0.0, 0.0, 1.3]},
+ {"id": 14, "title": "t14", "info": {"year": 2014}, "embedding": [0.0, 0.0, 0.0, 1.4]},
+ {"id": 15, "title": "t15", "info": {"year": 2015}, "embedding": [0.0, 0.0, 0.0, 1.5]},
+ {"id": 16, "title": "t16", "info": {"year": 2016}, "embedding": [10.0, 10.0, 10.0, 0.1]},
+ {"id": 17, "title": "t17", "info": {"year": 2017}, "embedding": [10.0, 10.0, 10.0, 0.2]},
+ {"id": 18, "title": "t18", "info": {"year": 2018}, "embedding": [10.0, 10.0, 10.0, 0.3]},
+ {"id": 19, "title": "t19", "info": {"year": 2019}, "embedding": [10.0, 10.0, 10.0, 0.4]},
+ {"id": 20, "title": "t20", "info": {"year": 2020}, "embedding": [10.0, 10.0, 10.0, 0.5]},
+ {"id": 21, "title": "t21", "info": {"year": 2021}, "embedding": [10.0, 10.0, 10.0, 0.6]},
+ {"id": 22, "title": "t22", "info": {"year": 2022}, "embedding": [10.0, 10.0, 10.0, 0.7]},
+ {"id": 23, "title": "t23", "info": {"year": 2023}, "embedding": [10.0, 10.0, 10.0, 0.8]},
+ {"id": 24, "title": "t24", "info": {"year": 2024}, "embedding": [10.0, 10.0, 10.0, 0.9]},
+ {"id": 25, "title": "t25", "info": {"year": 2025}, "embedding": [10.0, 10.0, 10.0, 1.0]},
+ {"id": 26, "title": "t26", "info": {"year": 2026}, "embedding": [10.0, 10.0, 10.0, 1.1]},
+ {"id": 27, "title": "t27", "info": {"year": 2027}, "embedding": [10.0, 10.0, 10.0, 1.2]},
+ {"id": 28, "title": "t28", "info": {"year": 2028}, "embedding": [10.0, 10.0, 10.0, 1.3]},
+ {"id": 29, "title": "t29", "info": {"year": 2029}, "embedding": [10.0, 10.0, 10.0, 1.4]},
+ {"id": 30, "title": "t30", "info": {"year": 2030}, "embedding": [10.0, 10.0, 10.0, 1.5]}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.3.ddl.sqlpp
new file mode 100644
index 0000000000..8deeec9a1f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.3.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ANALYZE DATASET test.MovieSmall WITH {"sample-seed": 1000};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.4.ddl.sqlpp
new file mode 100644
index 0000000000..1d79f228a9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.4.ddl.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE INDEX vt_default ON MovieSmall(embedding VECTOR) TYPE VTREE
+ WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "seed": 42 };
+
+CREATE INDEX vt_none ON MovieSmall(embedding VECTOR) TYPE VTREE
+ WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "seed": 42,
+ "storage-block-compression": {"scheme": "none"} };
+
+CREATE INDEX vt_snappy ON MovieSmall(embedding VECTOR) TYPE VTREE
+ WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "seed": 42,
+ "storage-block-compression": {"scheme": "snappy"} };
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.5.query.sqlpp
new file mode 100644
index 0000000000..5cc65a095b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.5.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// The scheme is catalog state, not a vector parameter, so it appears only as BlockLevelStorageCompression.
+SELECT i.IndexName, i.BlockLevelStorageCompression
+FROM Metadata.`Index` i
+WHERE i.DataverseName = "test" AND i.DatasetName = "MovieSmall" AND i.IndexStructure = "VTREE"
+ORDER BY i.IndexName;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.99.ddl.sqlpp
new file mode 100644
index 0000000000..20dc6fde5e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.99.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-invalid-compression-scheme/create-index-vtree-invalid-compression-scheme.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-invalid-compression-scheme/create-index-vtree-invalid-compression-scheme.1.ddl.sqlpp
new file mode 100644
index 0000000000..6452b292f9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/create-index-vtree-invalid-compression-scheme/create-index-vtree-invalid-compression-scheme.1.ddl.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE MovieType AS { id: int, embedding: [double] };
+CREATE DATASET MovieSmall(MovieType) PRIMARY KEY id;
+
+CREATE INDEX vt ON MovieSmall(embedding VECTOR) TYPE VTREE
+ WITH { "dimension": 4, "similarity": "euclidean", "storage-block-compression": {"scheme": "zip"} };
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.5.adm
new file mode 100644
index 0000000000..2b48a83063
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-compression-scheme/create-index-vtree-compression-scheme.5.adm
@@ -0,0 +1,3 @@
+{ "IndexName": "vt_default", "BlockLevelStorageCompression": { "IndexCompressionScheme": "snappy" } }
+{ "IndexName": "vt_none" }
+{ "IndexName": "vt_snappy", "BlockLevelStorageCompression": { "IndexCompressionScheme": "snappy" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-metadata/create-index-vtree-metadata.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-metadata/create-index-vtree-metadata.5.adm
index 217e8f6997..31a123b80a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-metadata/create-index-vtree-metadata.5.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/create-index-vtree-metadata/create-index-vtree-metadata.5.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "MovieSmall", "IndexName": "idx_emb", "IndexStructure": "VTREE", "SearchKey": [ [ "embedding" ] ], "IsPrimary": false, "PendingOp": 0, "dimension": 4, "similarity": "cosine", "quantization": "SQ4", "train_list_fraction": 1.0, "epsilon": 0.5, "num_clusters": 2, "cross_pollination_m": 3, "rng_factor": 1.5, "seed": 42, "IncludeFields": [ [ "title" ], [ "info", "year" ] ] }
\ No newline at end of file
+{ "DataverseName": "test", "DatasetName": "MovieSmall", "IndexName": "idx_emb", "IndexStructure": "VTREE", "SearchKey": [ [ "embedding" ] ], "IsPrimary": false, "PendingOp": 0, "dimension": 4, "similarity": "cosine", "quantization": "SQ4", "train_list_fraction": 1.0, "epsilon": 0.5, "num_clusters": 2, "cross_pollination_m": 3, "rng_factor": 1.5, "seed": 42, "IncludeFields": [ [ "title" ], [ "info", "year" ] ], "BlockLevelStorageCompression": { "IndexCompressionScheme": "snappy" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
index e3dcee2a51..d3fed5eea5 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
@@ -81,11 +81,13 @@ public class CreateIndexStatement extends AbstractStatement {
this.castDefaultNull = OptionalBoolean.ofNullable(castDefaultNull);
this.castConfig = castConfig == null ? Collections.emptyMap() : castConfig;
this.includeElements = includeElements;
- // A vector index always has a WITH clause, so this is non-null exactly for TYPE VTREE. On a BTREE or ARRAY
- // index the clause carries the compression scheme only; the grammar rejects it on any other type.
+ // A vector index always has a WITH clause, so this is non-null exactly for TYPE VTREE; its clause may also
+ // name a compression scheme. On a BTREE or ARRAY index the clause carries the compression scheme only; the
+ // grammar rejects it on any other type.
this.vectorParameters =
indexType == IndexType.VTREE ? VectorIndexDeclUtil.validateAndGetParameters(withObjectRecord) : null;
- this.compressionScheme = indexType == IndexType.VTREE ? null
+ this.compressionScheme = indexType == IndexType.VTREE
+ ? DatasetDeclParametersUtil.validateAndGetVectorIndexCompressionScheme(withObjectRecord)
: DatasetDeclParametersUtil.validateAndGetIndexCompressionScheme(withObjectRecord);
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java
index d24a284ee5..b8b24c6f05 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/DatasetDeclParametersUtil.java
@@ -27,6 +27,7 @@ import org.apache.asterix.lang.common.expression.RecordTypeDefinition;
import org.apache.asterix.lang.common.expression.TypeExpression;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.object.base.AdmObjectNode;
+import org.apache.asterix.object.base.IAdmNode;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.AUnionType;
import org.apache.asterix.om.types.BuiltinType;
@@ -117,6 +118,25 @@ public class DatasetDeclParametersUtil {
: storageBlockCompression.getOptionalString(STORAGE_BLOCK_COMPRESSION_SCHEME_PARAMETER_NAME);
}
+ /**
+ * @return the compression scheme a vector index's WITH clause names, or null where it names none. The rest of
+ * that clause is vector parameters, validated by {@link VectorIndexDeclUtil}, so only this key is read.
+ */
+ public static String validateAndGetVectorIndexCompressionScheme(RecordConstructor withRecord)
+ throws CompilationException {
+ if (withRecord == null) {
+ return null;
+ }
+ IAdmNode storageBlockCompression =
+ ExpressionUtils.toNode(withRecord).get(STORAGE_BLOCK_COMPRESSION_PARAMETER_NAME);
+ if (storageBlockCompression == null) {
+ return null;
+ }
+ new ConfigurationTypeValidator().validateType(getStorageBlockCompressionType(), storageBlockCompression);
+ return ((AdmObjectNode) storageBlockCompression)
+ .getOptionalString(STORAGE_BLOCK_COMPRESSION_SCHEME_PARAMETER_NAME);
+ }
+
private static ARecordType getIndexWithObjectType() {
return new ARecordType("withObject", new String[] { STORAGE_BLOCK_COMPRESSION_PARAMETER_NAME },
new IAType[] { AUnionType.createUnknownableType(getStorageBlockCompressionType()) }, false);
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/VectorIndexDeclUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/VectorIndexDeclUtil.java
index 940f8c1e4c..23174d3ec0 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/VectorIndexDeclUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/VectorIndexDeclUtil.java
@@ -134,9 +134,13 @@ public class VectorIndexDeclUtil {
private static void validateWithClauseFieldNames(AdmObjectNode node) throws CompilationException {
for (String name : node.getFieldNames()) {
- if (!VectorIndexParameters.isKnown(name)) {
- throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED, "Unknown field `"
- + name + "` in WITH clause. Allowed fields: " + VectorIndexParameters.nameList());
+ // The compression scheme shares the clause but is not a vector parameter; CreateIndexStatement reads it.
+ if (!VectorIndexParameters.isKnown(name)
+ && !DatasetDeclParametersUtil.STORAGE_BLOCK_COMPRESSION_PARAMETER_NAME.equals(name)) {
+ throw new CompilationException(ErrorCode.COMPILATION_VECTOR_INDEX_CREATION_FAILED,
+ "Unknown field `" + name + "` in WITH clause. Allowed fields: "
+ + VectorIndexParameters.nameList() + ", "
+ + DatasetDeclParametersUtil.STORAGE_BLOCK_COMPRESSION_PARAMETER_NAME);
}
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/VTreeResourceFactoryProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/VTreeResourceFactoryProvider.java
index d7c1670bd7..05fdce3f70 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/VTreeResourceFactoryProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/VTreeResourceFactoryProvider.java
@@ -44,6 +44,7 @@ import org.apache.asterix.runtime.utils.VectorDistanceFunctionFactory;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
import org.apache.hyracks.algebricks.data.ITypeTraitProvider;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.data.std.accessors.DoubleBinaryComparatorFactory;
@@ -164,13 +165,17 @@ public class VTreeResourceFactoryProvider implements IResourceFactoryProvider {
// and throws otherwise. Under an atomic statement those writers are the normal state, and the
// throw unwinds into GlobalVirtualBufferCache$FlushThread, which halts the JVM.
boolean atomic = dataset.isAtomic();
+ // Same source as the B-tree secondaries: the scheme recorded on the index when it was created.
+ ICompressorDecompressorFactory compDecompFactory =
+ mdProvider.getCompressionManager().getFactory(index.getCompressionScheme());
return new LSMVTreeLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits,
filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory, pageWriteCallbackFactory,
metadataPageManagerFactory, vbcProvider, ioSchedulerProvider, mergePolicyFactory,
mergePolicyProperties, true, vectorDimensions, vectorFields,
typeTraitProvider.getTypeTrait(BuiltinType.ANULL), NullIntrospector.INSTANCE, atomic,
vectorAccessorFactory, VTreeDataTupleAccessor.identityFields(isQuantized, numPrimaryKeys),
- numIncludeFields, distanceFunctionFactory, quantizerFactory, crossPollination, epsilon);
+ numIncludeFields, distanceFunctionFactory, quantizerFactory, crossPollination, epsilon,
+ compDecompFactory);
} else {
return null;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResource.java
index 5166ccce1e..0a0cebad5b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResource.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.Objects;
import org.apache.hyracks.api.application.INCServiceContext;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.ErrorCode;
@@ -53,6 +54,7 @@ import org.apache.hyracks.storage.am.vector.impls.VTreeDataTupleBuilderFactory;
import org.apache.hyracks.storage.am.vector.utils.CrossPollinationConfig;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.IStorageManager;
+import org.apache.hyracks.storage.common.compression.NoOpCompressorDecompressorFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -81,6 +83,7 @@ public class LSMVTreeLocalResource extends LsmResource implements IQuantizedReso
private static final String KEY_CONFIDENCE_INTERVAL = "confidenceInterval";
private static final String KEY_BITS = "bits";
private static final String KEY_SAMPLE_COUNT = "sampleCount";
+ private static final String KEY_COMPRESSOR_DECOMPRESSOR_FACTORY = "compressorDecompressorFactory";
protected final int vectorDimensions;
protected final int[] vectorFields;
@@ -113,6 +116,12 @@ public class LSMVTreeLocalResource extends LsmResource implements IQuantizedReso
/** Level-wise candidate window from the DDL; persisted for the same reason as {@link #crossPollination}. */
protected final double epsilon;
+ /**
+ * Page compression for the data ({@code .vct}) components. Persisted per index rather than resolved from the
+ * dataset at open time, so an index built before VTree compression existed keeps reading as uncompressed.
+ */
+ protected final ICompressorDecompressorFactory compressorDecompressorFactory;
+
/**
* Scalar-quantization calibration, or {@code null} for a non-quantized index. Written once by
* {@link QuantizedIndexBuilder} between resource creation and the first {@link #createInstance}, so
@@ -131,7 +140,8 @@ public class LSMVTreeLocalResource extends LsmResource implements IQuantizedReso
ITypeTraits nullTypeTraits, INullIntrospector nullIntrospector, boolean atomic,
IVTreeBinaryAccessorFactory vectorAccessorFactory, int[] identityFields, int numIncludeFields,
IVTreeDistanceFunctionFactory distanceFunctionFactory, IVTreeQuantizerFactory quantizerFactory,
- CrossPollinationConfig crossPollination, double epsilon) {
+ CrossPollinationConfig crossPollination, double epsilon,
+ ICompressorDecompressorFactory compressorDecompressorFactory) {
super(path, storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields,
opTrackerProvider, ioOpCallbackFactory, pageWriteCallbackFactory, metadataPageManagerFactory,
vbcProvider, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, nullTypeTraits,
@@ -146,13 +156,16 @@ public class LSMVTreeLocalResource extends LsmResource implements IQuantizedReso
this.quantizerFactory = quantizerFactory;
this.crossPollination = Objects.requireNonNull(crossPollination, "crossPollination");
this.epsilon = epsilon;
+ this.compressorDecompressorFactory =
+ Objects.requireNonNull(compressorDecompressorFactory, "compressorDecompressorFactory");
}
protected LSMVTreeLocalResource(IPersistedResourceRegistry registry, JsonNode json, int vectorDimensions,
int[] vectorFields, boolean atomic, IVTreeBinaryAccessorFactory vectorAccessorFactory, int[] identityFields,
int numIncludeFields, IVTreeDistanceFunctionFactory distanceFunctionFactory,
IVTreeQuantizerFactory quantizerFactory, CrossPollinationConfig crossPollination, double epsilon,
- VTreeQuantizationParams quantization) throws HyracksDataException {
+ VTreeQuantizationParams quantization, ICompressorDecompressorFactory compressorDecompressorFactory)
+ throws HyracksDataException {
super(registry, json);
this.vectorDimensions = vectorDimensions;
this.vectorFields = vectorFields;
@@ -165,6 +178,8 @@ public class LSMVTreeLocalResource extends LsmResource implements IQuantizedReso
this.crossPollination = Objects.requireNonNull(crossPollination, "crossPollination");
this.epsilon = epsilon;
this.quantization = quantization;
+ this.compressorDecompressorFactory =
+ Objects.requireNonNull(compressorDecompressorFactory, "compressorDecompressorFactory");
}
@Override
@@ -198,7 +213,7 @@ public class LSMVTreeLocalResource extends LsmResource implements IQuantizedReso
ioSchedulerProvider.getIoScheduler(ncServiceCtx), ioOpCallbackFactory, pageWriteCallbackFactory,
vectorDimensions, vectorFields, filterFields, null, null, null, durable, metadataPageManagerFactory,
atomic, null, vectorAccessorFactory, identityFields, dataTupleBuilderFactory, quantization,
- distanceFunctionFactory, quantizerFactory, crossPollination, epsilon);
+ distanceFunctionFactory, quantizerFactory, crossPollination, epsilon, compressorDecompressorFactory);
}
@Override
@@ -230,6 +245,7 @@ public class LSMVTreeLocalResource extends LsmResource implements IQuantizedReso
if (quantizerFactory != null) {
json.set(KEY_QUANTIZER_FACTORY, quantizerFactory.toJson(registry));
}
+ json.set(KEY_COMPRESSOR_DECOMPRESSOR_FACTORY, compressorDecompressorFactory.toJson(registry));
if (quantization != null) {
ObjectNode quantizationNode = OBJECT_MAPPER.createObjectNode();
quantizationNode.put(KEY_MIN_QUANTILE, quantization.minQuantile());
@@ -263,6 +279,12 @@ public class LSMVTreeLocalResource extends LsmResource implements IQuantizedReso
CrossPollinationConfig crossPollination = new CrossPollinationConfig(
require(json, KEY_CROSS_POLLINATION_M).asInt(), require(json, KEY_RNG_FACTOR).asDouble());
+ // Absent on an index persisted before VTree compression existed: its components carry no look-aside file,
+ // so it must keep opening as uncompressed. This is the one key that deliberately bypasses require().
+ ICompressorDecompressorFactory compDecompFactory =
+ (ICompressorDecompressorFactory) registry.deserializeOrDefault(
+ json.get(KEY_COMPRESSOR_DECOMPRESSOR_FACTORY), NoOpCompressorDecompressorFactory.class);
+
VTreeQuantizationParams quantization = null;
if (json.has(KEY_QUANTIZATION)) {
JsonNode node = json.get(KEY_QUANTIZATION);
@@ -273,7 +295,7 @@ public class LSMVTreeLocalResource extends LsmResource implements IQuantizedReso
}
return new LSMVTreeLocalResource(registry, json, vectorDimensions, vectorFields, atomic, vectorAccessorFactory,
identityFields, numIncludeFields, distanceFunctionFactory, quantizerFactory, crossPollination, epsilon,
- quantization);
+ quantization, compDecompFactory);
}
/**
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResourceFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResourceFactory.java
index dce42e5970..0526903c25 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResourceFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/dataflow/LSMVTreeLocalResourceFactory.java
@@ -21,6 +21,7 @@ package org.apache.hyracks.storage.am.lsm.vector.dataflow;
import java.util.Map;
import java.util.Objects;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.io.FileReference;
@@ -61,6 +62,8 @@ public class LSMVTreeLocalResourceFactory extends LsmResourceFactory {
protected final CrossPollinationConfig crossPollination;
/** Level-wise candidate window supplied at DDL time; threaded onto {@link LSMVTreeLocalResource}. */
protected final double epsilon;
+ /** Page compression for the data components; the dataset's scheme, threaded onto the resource. */
+ protected final ICompressorDecompressorFactory compressorDecompressorFactory;
public LSMVTreeLocalResourceFactory(IStorageManager storageManager, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] cmpFactories, ITypeTraits[] filterTypeTraits,
@@ -73,7 +76,8 @@ public class LSMVTreeLocalResourceFactory extends LsmResourceFactory {
ITypeTraits nullTypeTraits, INullIntrospector nullIntrospector, boolean atomic,
IVTreeBinaryAccessorFactory vectorAccessorFactory, int[] identityFields, int numIncludeFields,
IVTreeDistanceFunctionFactory distanceFunctionFactory, IVTreeQuantizerFactory quantizerFactory,
- CrossPollinationConfig crossPollination, double epsilon) {
+ CrossPollinationConfig crossPollination, double epsilon,
+ ICompressorDecompressorFactory compressorDecompressorFactory) {
super(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields,
opTrackerFactory, ioOpCallbackFactory, pageWriteCallbackFactory, metadataPageManagerFactory,
vbcProvider, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, nullTypeTraits,
@@ -88,6 +92,8 @@ public class LSMVTreeLocalResourceFactory extends LsmResourceFactory {
this.vectorAccessorFactory = vectorAccessorFactory;
this.identityFields = identityFields;
this.numIncludeFields = numIncludeFields;
+ this.compressorDecompressorFactory =
+ Objects.requireNonNull(compressorDecompressorFactory, "compressorDecompressorFactory");
}
@Override
@@ -97,6 +103,6 @@ public class LSMVTreeLocalResourceFactory extends LsmResourceFactory {
pageWriteCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider,
mergePolicyFactory, mergePolicyProperties, durable, vectorDimensions, vectorFields, nullTypeTraits,
nullIntrospector, atomic, vectorAccessorFactory, identityFields, numIncludeFields,
- distanceFunctionFactory, quantizerFactory, crossPollination, epsilon);
+ distanceFunctionFactory, quantizerFactory, crossPollination, epsilon, compressorDecompressorFactory);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeFileManager.java
index 9e0831c4f6..88bb4246ae 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/impls/LSMVTreeFileManager.java
@@ -22,8 +22,11 @@ package org.apache.hyracks.storage.am.lsm.vector.impls;
import java.io.FilenameFilter;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
@@ -34,6 +37,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.IndexComponentFileReferenc
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMVTreeComponentFileReferences;
import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.common.compression.file.CompressedFileReference;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -57,15 +61,19 @@ public class LSMVTreeFileManager extends AbstractLSMIndexFileManager {
(dir, name) -> !name.startsWith(".") && name.endsWith(VCTREE_SUFFIX);
public LSMVTreeFileManager(IIOManager ioManager, FileReference file,
- TreeIndexFactory<? extends ITreeIndex> vTreeFactory) {
- super(ioManager, file, null);
+ TreeIndexFactory<? extends ITreeIndex> vTreeFactory,
+ ICompressorDecompressorFactory compressorDecompressorFactory) {
+ super(ioManager, file, null, compressorDecompressorFactory);
this.vTreeFactory = vTreeFactory;
}
@Override
public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
String baseName = getNextComponentSequence(vTreeFilter);
- return new LSMVTreeComponentFileReferences(baseDir.getChild(baseName + DELIMITER + VCTREE_SUFFIX), null, null,
+ // Only the data component is compressed. The shared static structure is written once by a separate
+ // builder and must open the same way under every component, compressed dataset or not.
+ return new LSMVTreeComponentFileReferences(
+ getCompressedFileReferenceIfAny(baseName + DELIMITER + VCTREE_SUFFIX), null, null,
baseDir.getChild(STATIC_STRUCTURE_SUFFIX));
}
@@ -76,7 +84,8 @@ public class LSMVTreeFileManager extends AbstractLSMIndexFileManager {
// component references the same shared file the flush path references — there is no per-merge
// ".staticstructure" file, and nothing ever creates one. It goes in the dedicated static-structure
// slot, not the bloom-filter slot.
- return new LSMVTreeComponentFileReferences(baseDir.getChild(baseName + DELIMITER + VCTREE_SUFFIX), null, null,
+ return new LSMVTreeComponentFileReferences(
+ getCompressedFileReferenceIfAny(baseName + DELIMITER + VCTREE_SUFFIX), null, null,
baseDir.getChild(STATIC_STRUCTURE_SUFFIX));
}
@@ -103,6 +112,7 @@ public class LSMVTreeFileManager extends AbstractLSMIndexFileManager {
}
if (allVTreeFiles.isEmpty()) {
+ cleanLookAsideFiles(Collections.emptySet(), vTreeFactory.getBufferCache());
return validFiles;
}
@@ -131,6 +141,12 @@ public class LSMVTreeFileManager extends AbstractLSMIndexFileManager {
}
}
+ Set<String> survivingSequences = new HashSet<>();
+ for (IndexComponentFileReference survivor : survivors) {
+ survivingSequences.add(survivor.getSequence());
+ }
+ cleanLookAsideFiles(survivingSequences, vTreeFactory.getBufferCache());
+
// LSM expects disk components ordered newest -> oldest: [2,2], [1,1], [0,0].
survivors.sort(Collections.reverseOrder());
for (IndexComponentFileReference vTreeFile : survivors) {
@@ -174,7 +190,17 @@ public class LSMVTreeFileManager extends AbstractLSMIndexFileManager {
* listing under-reports (and would silently drop valid components here).
*/
private void collectVTreeFiles(List<IndexComponentFileReference> files) throws HyracksDataException {
- for (FileReference fileRef : ioManager.list(baseDir, vTreeFilter)) {
+ for (FileReference listed : ioManager.list(baseDir, vTreeFilter)) {
+ // Re-resolve through the compression scheme, or a compressed component would be opened as a raw file.
+ FileReference fileRef = getCompressedFileReferenceIfAny(listed.getName());
+ if (fileRef.isCompressed()
+ && !ioManager.exists(((CompressedFileReference) fileRef).getLAFFileReference())) {
+ // A compressed component without its look-aside file cannot be read: a delete that removed the
+ // LAF but not the data file before a crash (CompressedFileManager.State.INVALID). Finish it.
+ LOGGER.log(Level.DEBUG, "Deleting VTree component {} with no look-aside file", fileRef);
+ cleanupOrphanedVTreeFile(fileRef);
+ continue;
+ }
files.add(IndexComponentFileReference.of(fileRef));
}
}
@@ -208,11 +234,14 @@ public class LSMVTreeFileManager extends AbstractLSMIndexFileManager {
*/
private void cleanupOrphanedVTreeFile(FileReference vTreeFile) {
try {
- LOGGER.log(Level.TRACE, "Cleaning up orphaned VTree file: {}", vTreeFile.getAbsolutePath());
- ioManager.delete(vTreeFile);
+ LOGGER.log(Level.DEBUG, "Cleaning up orphaned VTree file: {}", vTreeFile.getAbsolutePath());
+ // through the buffer cache, which deletes a compressed file's look-aside file first, as recovery
+ // relies on, and unmaps the file if it is open
+ delete(vTreeFactory.getBufferCache(), vTreeFile);
} catch (Exception e) {
LOGGER.log(Level.TRACE, "Failed to clean up orphaned VTree file {}: {}", vTreeFile.getAbsolutePath(),
e.getMessage());
}
}
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/utils/LSMVTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/utils/LSMVTreeUtils.java
index c562a61230..a5edafbcc1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/utils/LSMVTreeUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-vtree/src/main/java/org/apache/hyracks/storage/am/lsm/vector/utils/LSMVTreeUtils.java
@@ -21,6 +21,7 @@ package org.apache.hyracks.storage.am.lsm.vector.utils;
import java.util.List;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
@@ -98,6 +99,8 @@ public final class LSMVTreeUtils {
* would be a second source of truth for it. Callers that do not care
* about replication pass an explicit single-closest config.
* @param epsilon level-wise candidate window from the index DDL, for the same reason.
+ * @param compressorDecompressorFactory page compression for the data components; the shared static
+ * structure is never compressed.
*/
public static LSMVTree createLSMTree(NCConfig storageConfig, IIOManager ioManager,
List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
@@ -111,7 +114,8 @@ public final class LSMVTreeUtils {
IVTreeBinaryAccessorFactory vectorAccessorFactory, int[] identityFields,
IVTreeDataTupleBuilderFactory dataTupleBuilderFactory, VTreeQuantizationParams quantizationParams,
IVTreeDistanceFunctionFactory distanceFunctionFactory, IVTreeQuantizerFactory quantizerFactory,
- CrossPollinationConfig crossPollination, double epsilon) throws HyracksDataException {
+ CrossPollinationConfig crossPollination, double epsilon,
+ ICompressorDecompressorFactory compressorDecompressorFactory) throws HyracksDataException {
// VTree tuples contain no field that is both fixed-length AND nullable, so the null bitmap is never
// load-bearing and no INullIntrospector is needed (unlike BTree/RTree secondary keys):
@@ -179,7 +183,8 @@ public final class LSMVTreeUtils {
interiorFrameFactory, leafFrameFactory, metadataFrameFactory, insertDataFrameFactory, cmpFactories,
TREE_INDEX_FIELD_COUNT, vectorDimensions, vectorAccessorFactory, dataTupleBuilderFactory,
quantizationParams, distanceFunctionFactory, quantizerFactory, crossPollination, epsilon);
- ILSMIndexFileManager fileManager = new LSMVTreeFileManager(ioManager, file, vtreeFactory);
+ ILSMIndexFileManager fileManager =
+ new LSMVTreeFileManager(ioManager, file, vtreeFactory, compressorDecompressorFactory);
ILSMDiskComponentFactory componentFactory = new LSMVTreeDiskComponentFactory(vtreeFactory, filterHelper);
return new LSMVTree(storageConfig, ioManager, virtualBufferCaches, interiorFrameFactory, leafFrameFactory,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeNSMFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeNSMFrame.java
index bdec9eba50..3d92d73a6f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeNSMFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/frames/VTreeNSMFrame.java
@@ -19,9 +19,13 @@
package org.apache.hyracks.storage.am.vector.frames;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.btree.frames.OrderedSlotManager;
import org.apache.hyracks.storage.am.common.api.ISlotManager;
import org.apache.hyracks.storage.am.common.api.ISplitKey;
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
@@ -61,6 +65,9 @@ public abstract class VTreeNSMFrame extends TreeIndexNSMFrame implements IVTreeF
// Offset of the 4-byte centroid ID field. Subclass headers extend from CENTROID_ID_OFFSET + Integer.BYTES.
protected static final int CENTROID_ID_OFFSET = CLUSTER_ID_OFFSET + Integer.BYTES;
+ // Every VTree frame is built on an OrderedSlotManager, so one slot size serves all of them.
+ private static final int SLOT_SIZE = new OrderedSlotManager().getSlotSize();
+
protected MultiComparator cmp;
public VTreeNSMFrame(ITreeIndexTupleWriter tupleWriter, ISlotManager slotManager) {
@@ -80,6 +87,22 @@ public abstract class VTreeNSMFrame extends TreeIndexNSMFrame implements IVTreeF
return CENTROID_ID_OFFSET + Integer.BYTES;
}
+ /**
+ * Zeroes the gap between the tuple area and the slot array of a VTree page about to be written. The page
+ * goes to disk whole, and that gap otherwise holds whatever its buffer held before: a reused cache victim,
+ * or an in-memory page copied as is. VTree pages leave a large gap, since every cluster ends on a partly
+ * filled page, so this is stale data in the file and compresses far worse than zeros.
+ *
+ * @param buf a heap buffer holding any VTree frame
+ */
+ public static void zeroUnusedSpace(ByteBuffer buf) {
+ int from = buf.getInt(Constants.FREE_SPACE_OFFSET);
+ int to = buf.capacity() - buf.getInt(Constants.TUPLE_COUNT_OFFSET) * SLOT_SIZE;
+ if (from < to) {
+ Arrays.fill(buf.array(), buf.arrayOffset() + from, buf.arrayOffset() + to, (byte) 0);
+ }
+ }
+
@Override
public ITreeIndexTupleReference createTupleReference() {
return tupleWriter.createTupleReference();
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeBulkLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeBulkLoader.java
index dccdbd4f64..61590905a5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeBulkLoader.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeBulkLoader.java
@@ -41,6 +41,7 @@ import org.apache.hyracks.storage.am.vector.api.IVTreeFrame;
import org.apache.hyracks.storage.am.vector.api.IVTreeInteriorFrame;
import org.apache.hyracks.storage.am.vector.api.IVTreeLeafFrame;
import org.apache.hyracks.storage.am.vector.api.IVTreeMetadataFrame;
+import org.apache.hyracks.storage.am.vector.frames.VTreeNSMFrame;
import org.apache.hyracks.storage.am.vector.utils.VTreeDataTupleAccessor;
import org.apache.hyracks.storage.am.vector.utils.VTreeLeafNeighborList;
import org.apache.hyracks.storage.am.vector.utils.VTreeMetadataKeys;
@@ -647,6 +648,7 @@ public class VTreeBulkLoader extends PageWriteFailureCallback implements IIndexB
}
private void write(ICachedPage cPage) throws HyracksDataException {
+ VTreeNSMFrame.zeroUnusedSpace(cPage.getBuffer());
compressedPageWriter.prepareWrite(cPage);
pageWriter.write(cPage);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeFlushLoader.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeFlushLoader.java
index d40b3a8736..45efa2f9e0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeFlushLoader.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeFlushLoader.java
@@ -27,6 +27,7 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex;
import org.apache.hyracks.storage.am.vector.api.IVTreeInteriorFrame;
import org.apache.hyracks.storage.am.vector.api.IVTreeLeafFrame;
+import org.apache.hyracks.storage.am.vector.frames.VTreeNSMFrame;
import org.apache.hyracks.storage.am.vector.utils.VTreeLeafNeighborList;
import org.apache.hyracks.storage.am.vector.utils.VTreeMetadataKeys;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
@@ -249,6 +250,7 @@ public class VTreeFlushLoader extends PageWriteFailureCallback implements IIndex
}
private void write(ICachedPage cPage) throws HyracksDataException {
+ VTreeNSMFrame.zeroUnusedSpace(cPage.getBuffer());
compressedPageWriter.prepareWrite(cPage);
pageWriter.write(cPage);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeStaticStructureBuilder.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeStaticStructureBuilder.java
index 509015cc61..611645aac3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeStaticStructureBuilder.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreeStaticStructureBuilder.java
@@ -39,6 +39,7 @@ import org.apache.hyracks.storage.am.vector.api.IVTreeFrame;
import org.apache.hyracks.storage.am.vector.api.IVTreeInteriorFrame;
import org.apache.hyracks.storage.am.vector.api.IVTreeLeafFrame;
import org.apache.hyracks.storage.am.vector.api.VTreeStaticTupleConstants;
+import org.apache.hyracks.storage.am.vector.frames.VTreeNSMFrame;
import org.apache.hyracks.storage.am.vector.utils.VTreeLeafNeighborList;
import org.apache.hyracks.storage.am.vector.utils.VTreeMetadataKeys;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
@@ -360,6 +361,8 @@ public class VTreeStaticStructureBuilder extends PageWriteFailureCallback implem
}
private void writePage(ICachedPage page) throws HyracksDataException {
+ // Leaves are re-read from this local write by resolveAndUploadLeafNeighbors(), so zeroing here covers both.
+ VTreeNSMFrame.zeroUnusedSpace(page.getBuffer());
if (currentLevel == numLevels - 1) {
// Leaf page: its neighbor lists are still provisional, so record each centroid's placement
// and publish the page LOCAL-ONLY. resolveAndUploadLeafNeighbors() later re-reads, resolves,
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeCompressedMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeCompressedMergeTest.java
new file mode 100644
index 0000000000..32e862f507
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeCompressedMergeTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.storage.common.compression.SnappyCompressorDecompressorFactory;
+
+/**
+ * {@link LSMVTreeMergeTest} with Snappy page compression on the data components: bulk load, two flushes and a
+ * merge all write through the look-aside file, search reads back through it, and reactivation rediscovers the
+ * merged component as compressed.
+ */
+public class LSMVTreeCompressedMergeTest extends LSMVTreeMergeTest {
+
+ @Override
+ protected ICompressorDecompressorFactory compressorDecompressorFactory() {
+ return new SnappyCompressorDecompressorFactory();
+ }
+
+ @Override
+ protected void verifyComponentFiles(FileReference indexDir) {
+ String[] components = indexDir.getFile().list((dir, name) -> name.endsWith("_vct"));
+ assertEquals("the merge leaves exactly one data component", 1, components.length);
+ assertTrue("the data component has its look-aside file",
+ new File(indexDir.getFile(), components[0] + ".dic").exists());
+ String[] lafs = indexDir.getFile().list((dir, name) -> name.endsWith(".dic"));
+ assertEquals("the merged-away inputs' look-aside files are gone", 1, lafs.length);
+ String[] staticStructureLaf = indexDir.getFile().list((dir, name) -> name.startsWith(".staticstructure."));
+ assertEquals("the static structure is never compressed", 0, staticStructureLaf.length);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeFileManagerCrashRecoveryTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeFileManagerCrashRecoveryTest.java
index ef08c41ce2..64da7658fd 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeFileManagerCrashRecoveryTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeFileManagerCrashRecoveryTest.java
@@ -26,11 +26,16 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.util.List;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
import org.apache.hyracks.storage.am.lsm.vector.impls.LSMVTreeFileManager;
import org.apache.hyracks.storage.am.lsm.vector.util.LSMVTreeTestHarness;
+import org.apache.hyracks.storage.common.compression.NoOpCompressorDecompressorFactory;
+import org.apache.hyracks.storage.common.compression.SnappyCompressorDecompressorFactory;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -52,6 +57,7 @@ public class LSMVTreeFileManagerCrashRecoveryTest {
private static final String VCT = "_vct";
private static final String STATIC_STRUCTURE = ".staticstructure";
+ private static final String LAF = ".dic";
private final LSMVTreeTestHarness harness = new LSMVTreeTestHarness();
@@ -78,7 +84,7 @@ public class LSMVTreeFileManagerCrashRecoveryTest {
create(baseDir, "0_2" + VCT); // merged component (range 0..2)
create(baseDir, STATIC_STRUCTURE);
- LSMVTreeFileManager fileManager = new LSMVTreeFileManager(harness.getIOManager(), baseDir, null);
+ LSMVTreeFileManager fileManager = fileManager(baseDir, NoOpCompressorDecompressorFactory.INSTANCE);
List<LSMComponentFileReferences> valid = fileManager.cleanupAndGetValidFiles();
// Only the merged component is valid.
@@ -105,7 +111,7 @@ public class LSMVTreeFileManagerCrashRecoveryTest {
create(baseDir, "2_2" + VCT);
create(baseDir, STATIC_STRUCTURE);
- LSMVTreeFileManager fileManager = new LSMVTreeFileManager(harness.getIOManager(), baseDir, null);
+ LSMVTreeFileManager fileManager = fileManager(baseDir, NoOpCompressorDecompressorFactory.INSTANCE);
List<LSMComponentFileReferences> valid = fileManager.cleanupAndGetValidFiles();
assertEquals("all three flushed components survive", 3, valid.size());
@@ -116,6 +122,70 @@ public class LSMVTreeFileManagerCrashRecoveryTest {
assertTrue(exists(baseDir, "2_2" + VCT));
}
+ /** A compressed merge leaves each input's look-aside file behind too; those go with their data files. */
+ @Test
+ public void compressedCrashAfterMergeDropsInputLookAsideFiles() throws Exception {
+ FileReference baseDir = harness.getFileReference();
+ baseDir.getFile().mkdirs();
+ for (String seq : new String[] { "0_0", "1_1", "0_1" }) {
+ create(baseDir, seq + VCT);
+ create(baseDir, seq + VCT + LAF);
+ }
+ create(baseDir, STATIC_STRUCTURE);
+
+ List<LSMComponentFileReferences> valid = compressedFileManager(baseDir).cleanupAndGetValidFiles();
+
+ assertEquals(1, valid.size());
+ assertTrue("survivor must open through its look-aside file",
+ valid.get(0).getInsertIndexFileReference().isCompressed());
+ assertTrue(exists(baseDir, "0_1" + VCT));
+ assertTrue(exists(baseDir, "0_1" + VCT + LAF));
+ for (String seq : new String[] { "0_0", "1_1" }) {
+ assertFalse(seq + " data must be deleted", exists(baseDir, seq + VCT));
+ assertFalse(seq + " look-aside file must be deleted", exists(baseDir, seq + VCT + LAF));
+ }
+ assertTrue("static structure is never compressed and must be kept", exists(baseDir, STATIC_STRUCTURE));
+ }
+
+ /**
+ * A half-finished delete leaves a data file without its look-aside file, or the reverse. Neither is readable,
+ * so both are finished off; an intact compressed component is kept.
+ */
+ @Test
+ public void compressedComponentWithoutPartnerIsDropped() throws Exception {
+ FileReference baseDir = harness.getFileReference();
+ baseDir.getFile().mkdirs();
+ create(baseDir, "0_0" + VCT);
+ create(baseDir, "0_0" + VCT + LAF);
+ create(baseDir, "1_1" + VCT); // look-aside file already deleted
+ create(baseDir, "2_2" + VCT + LAF); // data file already deleted
+ create(baseDir, STATIC_STRUCTURE);
+
+ List<LSMComponentFileReferences> valid = compressedFileManager(baseDir).cleanupAndGetValidFiles();
+
+ assertEquals(1, valid.size());
+ assertTrue(valid.get(0).getInsertIndexFileReference().getFile().getName().startsWith("0_0"));
+ assertFalse(exists(baseDir, "1_1" + VCT));
+ assertFalse(exists(baseDir, "2_2" + VCT + LAF));
+ assertTrue(exists(baseDir, "0_0" + VCT + LAF));
+ }
+
+ private LSMVTreeFileManager compressedFileManager(FileReference baseDir) {
+ return fileManager(baseDir, new SnappyCompressorDecompressorFactory());
+ }
+
+ /** The file manager deletes through the buffer cache, so it needs a factory that carries the real one. */
+ private LSMVTreeFileManager fileManager(FileReference baseDir, ICompressorDecompressorFactory compression) {
+ TreeIndexFactory<ITreeIndex> factory = new TreeIndexFactory<>(harness.getIOManager(),
+ harness.getDiskBufferCache(), null, null, null, null, 0) {
+ @Override
+ public ITreeIndex createIndexInstance(FileReference file) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ return new LSMVTreeFileManager(harness.getIOManager(), baseDir, factory, compression);
+ }
+
private static void create(FileReference baseDir, String name) throws Exception {
File f = baseDir.getChild(name).getFile();
f.getParentFile().mkdirs();
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeLocalResourceJsonTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeLocalResourceJsonTest.java
index 265ffae1f0..dde94dfcba 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeLocalResourceJsonTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeLocalResourceJsonTest.java
@@ -30,6 +30,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -49,6 +50,8 @@ import org.apache.hyracks.storage.am.vector.api.IVTreeQuantizerFactory;
import org.apache.hyracks.storage.am.vector.api.VTreeQuantizationParams;
import org.apache.hyracks.storage.am.vector.utils.CrossPollinationConfig;
import org.apache.hyracks.storage.common.IStorageManager;
+import org.apache.hyracks.storage.common.compression.NoOpCompressorDecompressorFactory;
+import org.apache.hyracks.storage.common.compression.SnappyCompressorDecompressorFactory;
import org.junit.Test;
import com.fasterxml.jackson.databind.JsonNode;
@@ -71,6 +74,7 @@ public class LSMVTreeLocalResourceJsonTest {
private static final int NUM_INCLUDE_FIELDS = 2;
private static final double EPSILON = 0.75;
private static final CrossPollinationConfig CROSS_POLLINATION = new CrossPollinationConfig(3, 1.0);
+ private static final ICompressorDecompressorFactory SNAPPY = new SnappyCompressorDecompressorFactory();
/** Every key {@code appendToJson} writes unconditionally, so every key {@code fromJson} may not default. */
private static final String[] REQUIRED_KEYS =
@@ -108,7 +112,8 @@ public class LSMVTreeLocalResourceJsonTest {
LSMVTreeLocalResource restored = (LSMVTreeLocalResource) LSMVTreeLocalResource.fromJson(registry, json);
assertEquals(text(json), text(restored.toJson(registry)));
- assertEquals(0, countOccurrences(text(resource(registry, null).toJson(registry)), "\"quantizerFactory\""));
+ assertEquals(0, countOccurrences(text(resource(registry, (IVTreeQuantizerFactory) null).toJson(registry)),
+ "\"quantizerFactory\""));
}
/** A truncated or foreign resource must be rejected, not silently defaulted into a mis-keyed index. */
@@ -129,6 +134,34 @@ public class LSMVTreeLocalResourceJsonTest {
}
}
+ /** The compression scheme is what decides whether a component opens through its look-aside file. */
+ @Test
+ public void compressedResourceRoundTrips() throws Exception {
+ FakeRegistry registry = new FakeRegistry();
+ JsonNode json = resource(registry, SNAPPY).toJson(registry);
+ LSMVTreeLocalResource restored = (LSMVTreeLocalResource) LSMVTreeLocalResource.fromJson(registry, json);
+
+ assertEquals(text(json), text(restored.toJson(registry)));
+ assertEquals(true, text(json).contains(SnappyCompressorDecompressorFactory.class.getName()));
+ }
+
+ /**
+ * A resource persisted before VTree compression has no compressor key. Its components carry no look-aside
+ * file, so it must come back uncompressed rather than being rejected like a missing required key.
+ */
+ @Test
+ public void resourceWithoutCompressorReadsAsUncompressed() throws Exception {
+ FakeRegistry registry = new FakeRegistry();
+ ObjectNode legacy = resource(registry, SNAPPY).toJson(registry).deepCopy();
+ legacy.remove("compressorDecompressorFactory");
+
+ LSMVTreeLocalResource restored = (LSMVTreeLocalResource) LSMVTreeLocalResource.fromJson(registry, legacy);
+
+ String restoredJson = text(restored.toJson(registry));
+ assertEquals(true, restoredJson.contains(NoOpCompressorDecompressorFactory.class.getName()));
+ assertEquals(false, restoredJson.contains(SnappyCompressorDecompressorFactory.class.getName()));
+ }
+
/** The base class owns filterFields; a subclass copy would write it twice and read back the later one. */
@Test
public void filterFieldsIsWrittenOnce() throws Exception {
@@ -154,10 +187,20 @@ public class LSMVTreeLocalResourceJsonTest {
}
private static LSMVTreeLocalResource resource(FakeRegistry registry) {
- return resource(registry, TestVTreeQuantizerFactory.INSTANCE);
+ return resource(registry, TestVTreeQuantizerFactory.INSTANCE, NoOpCompressorDecompressorFactory.INSTANCE);
}
private static LSMVTreeLocalResource resource(FakeRegistry registry, IVTreeQuantizerFactory quantizerFactory) {
+ return resource(registry, quantizerFactory, NoOpCompressorDecompressorFactory.INSTANCE);
+ }
+
+ private static LSMVTreeLocalResource resource(FakeRegistry registry,
+ ICompressorDecompressorFactory compressorDecompressorFactory) {
+ return resource(registry, TestVTreeQuantizerFactory.INSTANCE, compressorDecompressorFactory);
+ }
+
+ private static LSMVTreeLocalResource resource(FakeRegistry registry, IVTreeQuantizerFactory quantizerFactory,
+ ICompressorDecompressorFactory compressorDecompressorFactory) {
return new LSMVTreeLocalResource("dataverse/dataset/0/idx", registry.serializable(IStorageManager.class),
new ITypeTraits[] { registry.serializable(ITypeTraits.class) },
new IBinaryComparatorFactory[] { registry.serializable(IBinaryComparatorFactory.class) }, null, null,
@@ -168,7 +211,7 @@ public class LSMVTreeLocalResourceJsonTest {
registry.serializable(ILSMMergePolicyFactory.class), Collections.emptyMap(), true, VECTOR_DIMENSIONS,
VECTOR_FIELDS, null, null, true, TestDoubleArrayVectorAccessor.Factory.INSTANCE, IDENTITY_FIELDS,
NUM_INCLUDE_FIELDS, TestVTreeDistanceFunctionFactory.INSTANCE, quantizerFactory, CROSS_POLLINATION,
- EPSILON);
+ EPSILON, compressorDecompressorFactory);
}
/**
@@ -224,7 +267,7 @@ public class LSMVTreeLocalResourceJsonTest {
@Override
public IJsonSerializable deserializeOrDefault(JsonNode json, Class<? extends IJsonSerializable> clazz)
throws HyracksDataException {
- return json == null ? null : deserialize(json);
+ return json == null ? byClass.get(clazz.getName()) : deserialize(json);
}
FakeRegistry() {
@@ -236,6 +279,8 @@ public class LSMVTreeLocalResourceJsonTest {
TestDoubleArrayVectorAccessor.Factory.INSTANCE);
byClass.put(TestVTreeDistanceFunctionFactory.class.getName(), TestVTreeDistanceFunctionFactory.INSTANCE);
byClass.put(TestVTreeQuantizerFactory.class.getName(), TestVTreeQuantizerFactory.INSTANCE);
+ byClass.put(NoOpCompressorDecompressorFactory.class.getName(), NoOpCompressorDecompressorFactory.INSTANCE);
+ byClass.put(SnappyCompressorDecompressorFactory.class.getName(), SNAPPY);
}
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeMergeTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeMergeTest.java
index fe2f29a3a9..dcce1619df 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeMergeTest.java
@@ -24,8 +24,10 @@ import static org.junit.Assert.*;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
@@ -50,6 +52,7 @@ import org.apache.hyracks.storage.am.vector.VectorTreeTestUtils;
import org.apache.hyracks.storage.am.vector.impls.VTreeSearchPredicate;
import org.apache.hyracks.storage.common.IIndexAccessor;
import org.apache.hyracks.storage.common.IIndexCursor;
+import org.apache.hyracks.storage.common.compression.NoOpCompressorDecompressorFactory;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.After;
@@ -107,7 +110,8 @@ public class LSMVTreeMergeTest extends VectorIndexTestDriver {
harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(),
dataRecordSerdes, vectorDimension, harness.getMergePolicy(), harness.getOperationTracker(),
harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
- harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory());
+ harness.getPageWriteCallbackFactory(), harness.getMetadataPageManagerFactory(), null, 0,
+ compressorDecompressorFactory());
// Set test data in context
ctx.setStaticStructureCentroids(centroids);
@@ -156,11 +160,29 @@ public class LSMVTreeMergeTest extends VectorIndexTestDriver {
int queryK = 500;
verifyMergedRecords(ctx, queryVector, queryK);
LOGGER.info("Verification: Records from both components found after merge");
+
+ // Reactivation rediscovers the merged component from disk through the file manager's recovery path.
+ ctx.getIndex().deactivate();
+ ctx.getIndex().activate();
+ assertEquals("Should rediscover 1 disk component on reactivation", 1, lsmvTree.getDiskComponents().size());
+ verifyMergedRecords(ctx, queryVector, queryK);
+ verifyComponentFiles(harness.getFileReference());
} finally {
ctx.getIndex().deactivate();
}
}
+ /** Page compression for the data components; subclasses override to run the same flow compressed. */
+ protected ICompressorDecompressorFactory compressorDecompressorFactory() {
+ return NoOpCompressorDecompressorFactory.INSTANCE;
+ }
+
+ /** Checks the on-disk files the merge left, once reactivation has reconciled them. */
+ protected void verifyComponentFiles(FileReference indexDir) {
+ String[] lafs = indexDir.getFile().list((dir, name) -> name.endsWith(".dic"));
+ assertEquals("an uncompressed index writes no look-aside file", 0, lafs == null ? 0 : lafs.length);
+ }
+
/**
* Flush the current memory component to disk.
*/
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/util/LSMVTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/util/LSMVTreeTestContext.java
index d7b1957c62..1065397c69 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/util/LSMVTreeTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/util/LSMVTreeTestContext.java
@@ -21,6 +21,7 @@ package org.apache.hyracks.storage.am.lsm.vector.util;
import java.util.List;
+import org.apache.hyracks.api.compression.ICompressorDecompressorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
@@ -57,6 +58,7 @@ import org.apache.hyracks.storage.am.vector.impls.VTreeDataTupleBuilderFactory;
import org.apache.hyracks.storage.am.vector.utils.CrossPollinationConfig;
import org.apache.hyracks.storage.am.vector.utils.VTreeDataTupleAccessor;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.compression.NoOpCompressorDecompressorFactory;
/**
* Test context for LSM Vector Clustering Tree tests.
@@ -148,12 +150,26 @@ public final class LSMVTreeTestContext extends AbstractVectorTreeTestContext {
metadataPageManagerFactory, null, numIncludeFields);
}
+ public static LSMVTreeTestContext create(NCConfig storageConfig, IIOManager ioManager,
+ List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
+ ISerializerDeserializer[] fieldSerdes, int numVectorFields, ILSMMergePolicy mergePolicy,
+ ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+ ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
+ IMetadataPageManagerFactory metadataPageManagerFactory,
+ IVTreeDataTupleBuilderFactory dataTupleBuilderFactory, int numIncludeFields) throws Exception {
+ return create(storageConfig, ioManager, virtualBufferCaches, file, diskBufferCache, fieldSerdes,
+ numVectorFields, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory, pageWriteCallbackFactory,
+ metadataPageManagerFactory, dataTupleBuilderFactory, numIncludeFields,
+ NoOpCompressorDecompressorFactory.INSTANCE);
+ }
+
/**
* Create a new LSMVTreeTestContext with a custom data tuple creator factory and include fields.
*
* @param dataTupleBuilderFactory the factory to use for creating data tuples,
* or null to use the default (standard) factory
* @param numIncludeFields number of include fields in the data record (0 = none)
+ * @param compressorDecompressorFactory page compression for the data components
*/
public static LSMVTreeTestContext create(NCConfig storageConfig, IIOManager ioManager,
List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache,
@@ -161,7 +177,8 @@ public final class LSMVTreeTestContext extends AbstractVectorTreeTestContext {
ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
ILSMIOOperationCallbackFactory ioOpCallbackFactory, ILSMPageWriteCallbackFactory pageWriteCallbackFactory,
IMetadataPageManagerFactory metadataPageManagerFactory,
- IVTreeDataTupleBuilderFactory dataTupleBuilderFactory, int numIncludeFields) throws Exception {
+ IVTreeDataTupleBuilderFactory dataTupleBuilderFactory, int numIncludeFields,
+ ICompressorDecompressorFactory compressorDecompressorFactory) throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
@@ -196,7 +213,7 @@ public final class LSMVTreeTestContext extends AbstractVectorTreeTestContext {
TestVTreeDistanceFunctionFactory.INSTANCE, // distanceFunctionFactory (test fixture)
// Identity quantizer: these fixtures store full-precision vectors, and the top-K cursor
// requires a quantizer. Inert on this bare leaf layout, which carries no quantized bytes.
- TestVTreeQuantizerFactory.INSTANCE, SINGLE_CLOSEST, EPSILON);
+ TestVTreeQuantizerFactory.INSTANCE, SINGLE_CLOSEST, EPSILON, compressorDecompressorFactory);
return new LSMVTreeTestContext(fieldSerdes, lsmVTree, numVectorFields);
}
@@ -241,7 +258,7 @@ public final class LSMVTreeTestContext extends AbstractVectorTreeTestContext {
quantizationParams, TestVTreeDistanceFunctionFactory.INSTANCE,
// No quantizer: this fixture's leaves carry full-precision bytes in the quantized field,
// so a quantized distance off them would not mean anything. Neighbor tests never ask for one.
- null, SINGLE_CLOSEST, EPSILON);
+ null, SINGLE_CLOSEST, EPSILON, NoOpCompressorDecompressorFactory.INSTANCE);
return new LSMVTreeTestContext(fieldSerdes, lsmVTree, numVectorFields);
}