You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2026/09/22 13:01:57 UTC
[L] Change in asterixdb[master]: [NO ISSUE][STO] Report every VTree write to the transaction layer
From Hongyu Shi <le...@gmail.com>:
Hongyu Shi has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21864?usp=email )
Change subject: [NO ISSUE][STO] Report every VTree write to the transaction layer
......................................................................
[NO ISSUE][STO] Report every VTree write to the transaction layer
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
A VTree did not follow the secondary index protocol on abort. Every LSM secondary index reports each modification to the callback so the transaction layer logs it and undo can reverse it, but the VTree reported from only one page-level path, so inserts that split or allocated a page, and every delete, escaped the log: an aborted statement rolled back the primary index and every other secondary while the VTree kept its entry, which an index-only ANN query then returned as a live key. The report now comes from the two methods that run once per record, after the pages are released: once per record rather than per replica cluster, since deleting the record removes every replica, and off the latched path the callback's contract warns about.
Co-Authored-By: Claude Opus 5 <no...@anthropic.com>
Change-Id: I9c4c7be5e548600a461a40221d9950ab4ec4b30e
---
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/VectorQueries.xml
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.01.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.02.update.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.03.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.04.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.05.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.06.update.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.07.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.08.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.09.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.99.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.07.adm
A asterixdb/asterix-app/src/test/resources/runtimets/results/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.08.adm
A asterixdb/asterix-app/src/test/resources/runtimets/results/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.09.adm
M hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTree.java
M hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreePageMutator.java
A hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeModificationCallbackTest.java
17 files changed, 511 insertions(+), 7 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/64/21864/1
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/VectorQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/VectorQueries.xml
index babbaa7..e00c470 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/VectorQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/VectorQueries.xml
@@ -180,6 +180,14 @@
</compilation-unit>
</test-case>
<test-case FilePath="vector">
+ <!-- An insert that aborts must leave no entry behind in a vector index: the index-only ANN plan
+ has no primary lookup to filter one out, so a surviving entry is returned as a live key. -->
+ <compilation-unit name="vtree-abort-leaves-index-entry">
+ <output-dir compare="Clean-JSON">vtree-abort-leaves-index-entry</output-dir>
+ <expected-error>HYR0043: Index entry size</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="vector">
<!-- Embeddings the index cannot represent, arriving after the build on an open type so the
variable is typed ANY: null, a list holding a string, a non-list, and no field at all.
Each row stays in the collection and out of the index, and the accounting query from
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.01.ddl.sqlpp
new file mode 100644
index 0000000..fcbc9b9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.01.ddl.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// An insert that aborts must leave no trace in a VTree secondary index. The abort is provoked by a
+// second vector index whose INCLUDE column pushes its entry past the per-entry size bound, so the
+// first index has already written its entry when the statement fails.
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE MovieType AS {
+ id: int,
+ embedding: [double],
+ blob: string
+};
+
+CREATE DATASET Movie(MovieType) PRIMARY KEY id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.02.update.sqlpp
new file mode 100644
index 0000000..ca6f47a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.02.update.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+INSERT INTO Movie ([
+ {"id":1,"embedding":[0.0,0.0,0.0,0.10],"blob":"x"},
+ {"id":2,"embedding":[0.0,0.0,0.0,0.20],"blob":"x"},
+ {"id":3,"embedding":[0.0,0.0,0.0,0.30],"blob":"x"},
+ {"id":4,"embedding":[0.0,0.0,0.0,0.40],"blob":"x"},
+ {"id":5,"embedding":[0.0,0.0,0.0,0.50],"blob":"x"},
+ {"id":6,"embedding":[0.0,0.0,0.0,0.60],"blob":"x"},
+ {"id":7,"embedding":[0.0,0.0,0.0,0.70],"blob":"x"},
+ {"id":8,"embedding":[0.0,0.0,0.0,0.80],"blob":"x"},
+ {"id":9,"embedding":[0.0,0.0,0.0,0.90],"blob":"x"},
+ {"id":10,"embedding":[0.0,0.0,0.0,1.00],"blob":"x"},
+ {"id":11,"embedding":[0.0,0.0,0.0,1.10],"blob":"x"},
+ {"id":12,"embedding":[0.0,0.0,0.0,1.20],"blob":"x"},
+ {"id":13,"embedding":[0.0,0.0,0.0,1.30],"blob":"x"},
+ {"id":14,"embedding":[0.0,0.0,0.0,1.40],"blob":"x"},
+ {"id":15,"embedding":[0.0,0.0,0.0,1.50],"blob":"x"},
+ {"id":16,"embedding":[0.0,0.0,0.0,1.60],"blob":"x"}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.03.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.03.ddl.sqlpp
new file mode 100644
index 0000000..de4af96
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.03.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+ANALYZE DATASET Movie;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.04.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.04.ddl.sqlpp
new file mode 100644
index 0000000..1ddb38f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.04.ddl.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// The index under test. Its name sorts before the failing one, and the insert pipeline chains
+// same-type indexes in index-name order, so this one is written first.
+USE test;
+
+CREATE INDEX a_emb
+ ON Movie(embedding VECTOR)
+ TYPE VTREE
+ WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "seed": 42 };
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.05.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.05.ddl.sqlpp
new file mode 100644
index 0000000..4d9c675
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.05.ddl.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// The index that makes the insert fail, once its INCLUDE column makes the entry too wide.
+USE test;
+
+CREATE INDEX z_blob
+ ON Movie(embedding VECTOR)
+ INCLUDE (blob)
+ TYPE VTREE
+ WITH { "dimension": 4, "similarity": "euclidean", "num_clusters": 2, "seed": 42 };
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.06.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.06.update.sqlpp
new file mode 100644
index 0000000..d7c8232
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.06.update.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Fails on z_blob after a_emb has taken its entry, so the statement aborts.
+USE test;
+
+INSERT INTO Movie ({"id":99,"embedding":[0.0,0.0,0.0,0.01],"blob":repeat("a",20000)});
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.07.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.07.query.sqlpp
new file mode 100644
index 0000000..0b1c6bf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.07.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// Index-only ANN: a PK-only projection under a LIMIT reads the vector index alone, with no primary
+// lookup to filter what the index returns. Id 99 is nearer the origin than any committed row, so a
+// surviving entry shows up first.
+USE test;
+
+SELECT VALUE m.id
+FROM Movie m
+ORDER BY ann_distance(m.embedding, [0.0, 0.0, 0.0, 0.0], "euclidean", 1.0)
+LIMIT 5;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.08.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.08.query.sqlpp
new file mode 100644
index 0000000..16f7884
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.08.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE count(*) FROM Movie;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.09.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.09.query.sqlpp
new file mode 100644
index 0000000..f6d868e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.09.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// The aborted row is absent from the primary index, which is what makes a hit in 07 a phantom.
+USE test;
+
+SELECT VALUE m.id FROM Movie m WHERE m.id = 99;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.99.ddl.sqlpp
new file mode 100644
index 0000000..20dc6fd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.99.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.07.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.07.adm
new file mode 100644
index 0000000..8a1218a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.07.adm
@@ -0,0 +1,5 @@
+1
+2
+3
+4
+5
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.08.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.08.adm
new file mode 100644
index 0000000..b6a7d89
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.08.adm
@@ -0,0 +1 @@
+16
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.09.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.09.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/vector/vtree-abort-leaves-index-entry/vtree-abort-leaves-index-entry.09.adm
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTree.java
index 3c3e068..581a47c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTree.java
@@ -232,6 +232,11 @@
clusterResult.centroidId, tuple, ctx, getFileId());
}
}
+ // One report per record, not per page write: cross-pollination writes a replica per accepted
+ // cluster, and each replica may take a different page path. The transaction layer logs one record
+ // per call, and a single one is what an abort needs, since deleting the record removes every
+ // replica. Reported after the pages are released, so the callback never logs under a latch.
+ ctx.getModificationCallback().found(null, tuple);
}
/**
@@ -271,6 +276,10 @@
}
}
}
+ // As on the insert path. The previous version is reported as absent deliberately: undo restores a
+ // reported old version with an upsert, which a VTree does not support, so it must instead take the
+ // physical-delete route that removes the antimatter this delete just wrote.
+ ctx.getModificationCallback().found(null, tuple);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreePageMutator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreePageMutator.java
index 419a81f..4303e18 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreePageMutator.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-vtree/src/main/java/org/apache/hyracks/storage/am/vector/impls/VTreePageMutator.java
@@ -197,13 +197,13 @@
switch (spaceStatus) {
case SUFFICIENT_CONTIGUOUS_SPACE:
- insertSortedIntoDataPage(dataTuple, distance, dataPageId, originalTuple, ctx);
+ insertSortedIntoDataPage(dataTuple, distance, dataPageId, ctx);
return;
case SUFFICIENT_SPACE:
// Free bytes exist but are fragmented: a delete returns space to the total without
// moving FREE_SPACE_OFFSET back. Compact first, as BTreeNSMLeafFrame does.
ctx.getDataFrame().compact();
- insertSortedIntoDataPage(dataTuple, distance, dataPageId, originalTuple, ctx);
+ insertSortedIntoDataPage(dataTuple, distance, dataPageId, ctx);
return;
case INSUFFICIENT_SPACE:
// The split recomputes the insertion index in whichever half the tuple lands.
@@ -236,15 +236,14 @@
}
/**
- * Insert a data tuple into the currently-latched data frame at its key-sorted position, fire the
- * modification callback and bump the page LSN. Shared by the contiguous-space and post-compaction
- * insert paths in {@link #insertIntoDataPage}, which has already removed any entry the tuple replaces.
+ * Insert a data tuple into the currently-latched data frame at its key-sorted position and bump the
+ * page LSN. Shared by the contiguous-space and post-compaction insert paths in
+ * {@link #insertIntoDataPage}, which has already removed any entry the tuple replaces.
*/
private void insertSortedIntoDataPage(ITupleReference dataTuple, double distance, long dataPageId,
- ITupleReference originalTuple, VTreeOpContext ctx) throws HyracksDataException {
+ VTreeOpContext ctx) throws HyracksDataException {
VTreeDataFrame dataFrame = (VTreeDataFrame) ctx.getDataFrame();
dataFrame.insert(dataTuple, dataFrame.findInsertPosition(dataFrame.keyOf(dataTuple)));
- ctx.getModificationCallback().found(null, originalTuple);
ctx.getDataFrame().setPageLsn(ctx.getDataFrame().getPageLsn() + 1);
// No separator to maintain: a key past the page's maximum was routed here by the catch-all, so a
// lagging separator costs nothing, as a BTree reaches its rightmost child without one. A split
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeModificationCallbackTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeModificationCallbackTest.java
new file mode 100644
index 0000000..294264d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-vtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/vector/LSMVTreeModificationCallbackTest.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.dataflow.common.data.marshalling.DoubleArraySerializerDeserializer;
+import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import org.apache.hyracks.storage.am.common.TestOperationCallback;
+import org.apache.hyracks.storage.am.common.api.IExtendedModificationOperationCallback;
+import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
+import org.apache.hyracks.storage.am.lsm.vector.util.LSMVTreeTestContext;
+import org.apache.hyracks.storage.am.lsm.vector.util.LSMVTreeTestHarness;
+import org.apache.hyracks.storage.am.lsm.vector.util.VectorTestStructure;
+import org.apache.hyracks.storage.am.vector.AbstractVectorTreeTestContext;
+import org.apache.hyracks.storage.am.vector.TestDoubleArrayVectorAccessor;
+import org.apache.hyracks.storage.am.vector.VectorTreeTestUtils;
+import org.apache.hyracks.storage.am.vector.api.IVTreeBinaryAccessorFactory;
+import org.apache.hyracks.storage.common.IIndexAccessor;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * A VTree must report every tuple it writes to the modification callback, because that callback is what
+ * writes the transaction log record an abort later reverses. A write that goes unreported cannot be rolled
+ * back, so a statement that fails leaves the entry in the index forever, pointing at a row the primary
+ * index no longer has.
+ * <p>
+ * The counts are what matter here, not the tuple contents: one report per logical operation, whichever
+ * page path the write happens to take.
+ */
+public class LSMVTreeModificationCallbackTest {
+
+ private final LSMVTreeTestHarness harness = new LSMVTreeTestHarness();
+ private final VectorTreeTestUtils testUtils = new VectorTreeTestUtils();
+
+ /** Records of one leaf centroid of {@link VectorTestStructure#threeDim3Level}, so they share a cluster. */
+ private static final double[] CLUSTER_VECTOR = { 20.0, 30.0, 20.0 };
+
+ /** Enough same-cluster records to fill and split a 512-byte data page several times over. */
+ private static final int MANY = 40;
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ /**
+ * The first insert into a cluster whose memory component holds no data page yet. This is the path a
+ * freshly built index takes on its very first incremental insert, and the one that went unreported.
+ */
+ @Test
+ public void firstInsertIntoAClusterIsReported() throws Exception {
+ AbstractVectorTreeTestContext ctx = newActivatedContext();
+ try {
+ RecordingCallback callback = new RecordingCallback();
+ accessorWith(ctx, callback).insert(tuple(CLUSTER_VECTOR, "pk_first"));
+
+ assertEquals("one insert must be reported once", 1, callback.found.size());
+ assertNull("an insert has no previous version", callback.found.get(0).before);
+ assertNotNull(callback.found.get(0).after);
+ } finally {
+ ctx.getIndex().deactivate();
+ }
+ }
+
+ /**
+ * Enough inserts into one cluster to allocate a data page, fill it and split it repeatedly, so the
+ * count covers every page path rather than only the one a small test would reach.
+ */
+ @Test
+ public void everyInsertIsReportedOnce() throws Exception {
+ AbstractVectorTreeTestContext ctx = newActivatedContext();
+ try {
+ RecordingCallback callback = new RecordingCallback();
+ IIndexAccessor accessor = accessorWith(ctx, callback);
+ for (int i = 0; i < MANY; i++) {
+ accessor.insert(tuple(offsetOf(i), "pk_" + i));
+ }
+
+ assertEquals("every insert must be reported exactly once", MANY, callback.found.size());
+ } finally {
+ ctx.getIndex().deactivate();
+ }
+ }
+
+ /**
+ * A delete is reversible only if it is reported too, whether it cancels a record with antimatter or
+ * removes one outright.
+ */
+ @Test
+ public void everyDeleteIsReportedOnce() throws Exception {
+ AbstractVectorTreeTestContext ctx = newActivatedContext();
+ try {
+ IIndexAccessor writer = accessorWith(ctx, new RecordingCallback());
+ for (int i = 0; i < MANY; i++) {
+ writer.insert(tuple(offsetOf(i), "pk_" + i));
+ }
+
+ RecordingCallback callback = new RecordingCallback();
+ IIndexAccessor accessor = accessorWith(ctx, callback);
+ for (int i = 0; i < MANY; i++) {
+ accessor.delete(tuple(offsetOf(i), "pk_" + i));
+ }
+
+ assertEquals("every delete must be reported exactly once", MANY, callback.found.size());
+ } finally {
+ ctx.getIndex().deactivate();
+ }
+ }
+
+ /** Spreads records slightly along one axis so they land in one cluster but at distinct distances. */
+ private static double[] offsetOf(int i) {
+ return new double[] { CLUSTER_VECTOR[0] + i * 0.01, CLUSTER_VECTOR[1], CLUSTER_VECTOR[2] };
+ }
+
+ private AbstractVectorTreeTestContext newActivatedContext() throws Exception {
+ VectorTestStructure struct = VectorTestStructure.threeDim3Level();
+ AbstractVectorTreeTestContext ctx = LSMVTreeTestContext.create(harness.getNcConfig(), harness.getIOManager(),
+ harness.getVirtualBufferCaches(), harness.getFileReference(), harness.getDiskBufferCache(),
+ struct.getDataRecordSerdes(VectorTestStructure.BulkLoadRecordFormat.NAIVE), struct.getVectorDimension(),
+ harness.getMergePolicy(), harness.getOperationTracker(), harness.getIOScheduler(),
+ harness.getIOOperationCallbackFactory(), harness.getPageWriteCallbackFactory(),
+ harness.getMetadataPageManagerFactory());
+ ctx.setStaticStructureCentroids(struct.buildCentroidTuples());
+ ctx.setNumClustersPerLevel(struct.getNumClustersPerLevel());
+ ctx.setNumCentroidsPerLevel(struct.getCentroidsPerCluster());
+ ctx.getIndex().create();
+ ctx.getIndex().activate();
+ // The tree cannot route a write until its centroids are attached.
+ testUtils.buildStaticStructure(ctx);
+ return ctx;
+ }
+
+ private static IIndexAccessor accessorWith(AbstractVectorTreeTestContext ctx,
+ IExtendedModificationOperationCallback callback) throws HyracksDataException {
+ IndexAccessParameters iap = new IndexAccessParameters(callback, TestOperationCallback.INSTANCE);
+ iap.getParameters().put(IVTreeBinaryAccessorFactory.IAP_KEY, TestDoubleArrayVectorAccessor.Factory.INSTANCE);
+ return ctx.getIndex().createAccessor(iap);
+ }
+
+ private static ITupleReference tuple(double[] vector, String primaryKey) throws HyracksDataException {
+ try {
+ ArrayTupleBuilder builder = new ArrayTupleBuilder(2);
+ DoubleArraySerializerDeserializer.INSTANCE.serialize(vector, builder.getDataOutput());
+ builder.addFieldEndOffset();
+ new UTF8StringSerializerDeserializer().serialize(primaryKey, builder.getDataOutput());
+ builder.addFieldEndOffset();
+ ArrayTupleReference ref = new ArrayTupleReference();
+ ref.reset(builder.getFieldEndOffsets(), builder.getByteArray());
+ return ref;
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ /** Keeps one entry per {@code found} call; the transaction layer writes one log record per call. */
+ private static final class RecordingCallback implements IExtendedModificationOperationCallback {
+
+ private static final class Report {
+ private final ITupleReference before;
+ private final ITupleReference after;
+
+ private Report(ITupleReference before, ITupleReference after) {
+ this.before = before;
+ this.after = after;
+ }
+ }
+
+ private final List<Report> found = new ArrayList<>();
+
+ @Override
+ public void before(ITupleReference tuple) throws HyracksDataException {
+ // The real callbacks lock here; nothing to record.
+ }
+
+ @Override
+ public void found(ITupleReference before, ITupleReference after) throws HyracksDataException {
+ found.add(new Report(before, after));
+ }
+
+ @Override
+ public void after(ITupleReference after) throws HyracksDataException {
+ // Ancillary structures only.
+ }
+ }
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21864?usp=email
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I9c4c7be5e548600a461a40221d9950ab4ec4b30e
Gerrit-Change-Number: 21864
Gerrit-PatchSet: 1
Gerrit-Owner: Hongyu Shi <le...@gmail.com>