You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@trafficserver.apache.org by "masaori335 (via GitHub)" <gi...@apache.org> on 2026/10/16 01:41:26 UTC
[PR] Fix dir entry cleanup after a failed aggregation write [trafficserver]
masaori335 opened a new pull request, #13847:
URL: https://github.com/apache/trafficserver/pull/13847
Address an issue found at #13840 review.
The aggWriteDone() error branch set the stripe-relative block offset of each entry to delete from a raw byte position, so remove() never matched and the failed docs' entries outlived the write. It also looked up only the doc's key, which misses head entries registered under first_key.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@trafficserver.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fix dir entry cleanup after a failed aggregation write [trafficserver]
Posted by "Copilot (via GitHub)" <gi...@apache.org>.
Copilot commented on code in PR #13847:
URL: https://github.com/apache/trafficserver/pull/13847#discussion_r4293072933
##########
src/iocore/cache/unit_tests/test_AggWriteDone.cc:
##########
@@ -0,0 +1,146 @@
+/** @file
+
+ Unit tests for StripeSM::aggWriteDone.
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "main.h"
+#include "test_doubles.h"
+
+#include "../P_CacheDir.h"
+#include "../P_CacheInternal.h"
+
+#include <array>
+#include <cerrno>
+
+// Required by main.h
+int cache_vols = 1;
+bool reuse_existing_cache = false;
+
+namespace
+{
+
+void
+init_writer(FakeVC &vc, StripeSM *stripe)
+{
+ // Sharing the stripe mutex lets aggWrite(EVENT_CALL) return the writer instead of scheduling a callback to it.
+ vc.mutex = stripe->mutex;
+ vc.stripe = stripe;
+ rand_CacheKey(&vc.key);
+ vc.set_test_data("yay", 4);
+ vc.set_write_len(4);
+ vc.set_agg_len(stripe->round_to_approx_size(vc.write_len + vc.header_len + vc.frag_len + sizeof(Doc)));
+}
+
+// The doc of a head writer carries earliest_key, but openWriteCloseHeadDone registers its entry under first_key.
+void
+init_head_writer(FakeVC &vc, StripeSM *stripe)
+{
+ init_writer(vc, stripe);
+ vc.f.use_first_key = 1;
+ vc.earliest_key = vc.key;
+ rand_CacheKey(&vc.first_key);
Review Comment:
This test relies on randomly generated `CacheKey`s, which makes it non-deterministic and can introduce extremely rare flakiness (e.g., collisions or differing behavior across runs). Consider using fixed, explicitly constructed keys (or a deterministically seeded generator local to the test) so the test is fully reproducible.
##########
src/iocore/cache/StripeSM.cc:
##########
@@ -758,8 +758,10 @@ StripeSM::aggWriteDone(int event, Event *e)
dir_clear(&del_dir);
for (int done = 0; done < this->_write_buffer.get_buffer_pos();) {
Doc *doc = reinterpret_cast<Doc *>(this->_write_buffer.get_buffer() + done);
- dir_set_offset(&del_dir, directory.header->write_pos + done);
+ dir_set_offset(&del_dir, this->offset_to_vol_offset(directory.header->write_pos + done));
+ // A head's entry is under first_key rather than the doc's key, and the doc doesn't say which it is, so try both.
this->directory.remove(&doc->key, this, &del_dir);
+ this->directory.remove(&doc->first_key, this, &del_dir);
Review Comment:
Optional: if `doc->first_key` can be the same as `doc->key` for non-head docs, this will perform two identical directory lookups/removals. Consider guarding the second `remove()` with a key comparison so it only runs when the keys differ.
##########
src/iocore/cache/unit_tests/test_AggWriteDone.cc:
##########
@@ -0,0 +1,146 @@
+/** @file
+
+ Unit tests for StripeSM::aggWriteDone.
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "main.h"
+#include "test_doubles.h"
+
+#include "../P_CacheDir.h"
+#include "../P_CacheInternal.h"
+
+#include <array>
+#include <cerrno>
+
+// Required by main.h
+int cache_vols = 1;
+bool reuse_existing_cache = false;
+
+namespace
+{
+
+void
+init_writer(FakeVC &vc, StripeSM *stripe)
+{
+ // Sharing the stripe mutex lets aggWrite(EVENT_CALL) return the writer instead of scheduling a callback to it.
+ vc.mutex = stripe->mutex;
+ vc.stripe = stripe;
+ rand_CacheKey(&vc.key);
+ vc.set_test_data("yay", 4);
Review Comment:
This test relies on randomly generated `CacheKey`s, which makes it non-deterministic and can introduce extremely rare flakiness (e.g., collisions or differing behavior across runs). Consider using fixed, explicitly constructed keys (or a deterministically seeded generator local to the test) so the test is fully reproducible.
##########
src/iocore/cache/unit_tests/test_AggWriteDone.cc:
##########
@@ -0,0 +1,146 @@
+/** @file
+
+ Unit tests for StripeSM::aggWriteDone.
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "main.h"
+#include "test_doubles.h"
+
+#include "../P_CacheDir.h"
+#include "../P_CacheInternal.h"
+
+#include <array>
+#include <cerrno>
+
+// Required by main.h
+int cache_vols = 1;
+bool reuse_existing_cache = false;
+
+namespace
+{
+
+void
+init_writer(FakeVC &vc, StripeSM *stripe)
+{
+ // Sharing the stripe mutex lets aggWrite(EVENT_CALL) return the writer instead of scheduling a callback to it.
+ vc.mutex = stripe->mutex;
+ vc.stripe = stripe;
+ rand_CacheKey(&vc.key);
+ vc.set_test_data("yay", 4);
+ vc.set_write_len(4);
+ vc.set_agg_len(stripe->round_to_approx_size(vc.write_len + vc.header_len + vc.frag_len + sizeof(Doc)));
+}
+
+// The doc of a head writer carries earliest_key, but openWriteCloseHeadDone registers its entry under first_key.
+void
+init_head_writer(FakeVC &vc, StripeSM *stripe)
+{
+ init_writer(vc, stripe);
+ vc.f.use_first_key = 1;
+ vc.earliest_key = vc.key;
+ rand_CacheKey(&vc.first_key);
+}
+
+CacheKey const *
+registered_key(FakeVC const &vc)
+{
+ if (vc.f.use_first_key) {
+ return &vc.first_key;
+ }
+ return &vc.key;
+}
+
+} // namespace
+
+class AggWriteErrorTest : public CacheInit
+{
+public:
+ int
+ cache_init_success_callback(int /* event ATS_UNUSED */, void * /* e ATS_UNUSED */) override
+ {
+ REQUIRE(CacheProcessor::IsCacheEnabled() == CacheInitState::INITIALIZED);
+ REQUIRE(gnstripes >= 1);
+
+ StripeSM *stripe = gstripes[0];
+ MUTEX_TRY_LOCK(lock, stripe->mutex, this_ethread());
+ if (!lock.is_locked()) {
+ CONT_SCHED_LOCK_RETRY(this);
+ return EVENT_DONE;
+ }
+
+ stripe->clear_dir();
+
+ std::array<FakeVC, 3> writers;
+ init_writer(writers[0], stripe); // case 1 - Sits at write_pos, the start of the buffer.
+ init_writer(writers[1], stripe); // case 2 - Sits past write_pos, so its offset includes its position in the buffer.
+ init_head_writer(writers[2], stripe); // case 3 - A head: registered under first_key, not its doc's key.
+ int agg_len = 0;
+ for (auto &vc : writers) {
+ REQUIRE(stripe->add_writer(&vc));
+ agg_len += vc.agg_len;
+ }
+
+ // Below the high water mark and with no sync writer, aggWrite fills the buffer without starting the disk write.
+ stripe->aggWrite(EVENT_CALL, nullptr);
+ REQUIRE(!stripe->is_io_in_progress());
+ REQUIRE(stripe->get_agg_buf_pos() == agg_len);
+
+ // A writer inserts its entry as soon as its doc is in the buffer, before the disk write completes.
Review Comment:
The comment says the writer inserts the entry, but the test is explicitly inserting the entries itself. Update the comment to reflect what the test is doing (e.g., that the test is simulating the post-buffer/pre-disk-write state by inserting entries before calling `aggWriteDone()`).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@trafficserver.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fix dir entry cleanup after a failed aggregation write [trafficserver]
Posted by "masaori335 (via GitHub)" <gi...@apache.org>.
masaori335 commented on code in PR #13847:
URL: https://github.com/apache/trafficserver/pull/13847#discussion_r4321595555
##########
src/iocore/cache/StripeSM.cc:
##########
@@ -758,8 +758,10 @@ StripeSM::aggWriteDone(int event, Event *e)
dir_clear(&del_dir);
for (int done = 0; done < this->_write_buffer.get_buffer_pos();) {
Doc *doc = reinterpret_cast<Doc *>(this->_write_buffer.get_buffer() + done);
- dir_set_offset(&del_dir, directory.header->write_pos + done);
+ dir_set_offset(&del_dir, this->offset_to_vol_offset(directory.header->write_pos + done));
+ // A head's entry is under first_key rather than the doc's key, and the doc doesn't say which it is, so try both.
this->directory.remove(&doc->key, this, &del_dir);
+ this->directory.remove(&doc->first_key, this, &del_dir);
Review Comment:
This case does happen. A non-HTTP open_write sets first_key = key (Cache.cc:524). But the first remove() has already deleted the entry, so the second one finds nothing. It runs only when a disk write fails, and there's only ever one entry at a given offset. A guard would add a branch and change nothing.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@trafficserver.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fix dir entry cleanup after a failed aggregation write [trafficserver]
Posted by "masaori335 (via GitHub)" <gi...@apache.org>.
masaori335 commented on code in PR #13847:
URL: https://github.com/apache/trafficserver/pull/13847#discussion_r4321592924
##########
src/iocore/cache/unit_tests/test_AggWriteDone.cc:
##########
@@ -0,0 +1,146 @@
+/** @file
+
+ Unit tests for StripeSM::aggWriteDone.
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "main.h"
+#include "test_doubles.h"
+
+#include "../P_CacheDir.h"
+#include "../P_CacheInternal.h"
+
+#include <array>
+#include <cerrno>
+
+// Required by main.h
+int cache_vols = 1;
+bool reuse_existing_cache = false;
+
+namespace
+{
+
+void
+init_writer(FakeVC &vc, StripeSM *stripe)
+{
+ // Sharing the stripe mutex lets aggWrite(EVENT_CALL) return the writer instead of scheduling a callback to it.
+ vc.mutex = stripe->mutex;
+ vc.stripe = stripe;
+ rand_CacheKey(&vc.key);
+ vc.set_test_data("yay", 4);
Review Comment:
The test can't fail because of the key values. remove() matches on bucket, tag and offset, and the three writers sit at different offsets in the buffer, so one writer's key can never remove another writer's entry. The only effect is this: if earliest_key and first_key happen to share the same bucket and tag, case 3 still passes but doesn't really test the first_key removal. The odds are about 1/(4096 × number of buckets), and it can never make the test fail. test_CacheDir.cc also uses rand_CacheKey, so it's the existing convention.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@trafficserver.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fix dir entry cleanup after a failed aggregation write [trafficserver]
Posted by "masaori335 (via GitHub)" <gi...@apache.org>.
masaori335 commented on code in PR #13847:
URL: https://github.com/apache/trafficserver/pull/13847#discussion_r4321598636
##########
src/iocore/cache/unit_tests/test_AggWriteDone.cc:
##########
@@ -0,0 +1,146 @@
+/** @file
+
+ Unit tests for StripeSM::aggWriteDone.
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "main.h"
+#include "test_doubles.h"
+
+#include "../P_CacheDir.h"
+#include "../P_CacheInternal.h"
+
+#include <array>
+#include <cerrno>
+
+// Required by main.h
+int cache_vols = 1;
+bool reuse_existing_cache = false;
+
+namespace
+{
+
+void
+init_writer(FakeVC &vc, StripeSM *stripe)
+{
+ // Sharing the stripe mutex lets aggWrite(EVENT_CALL) return the writer instead of scheduling a callback to it.
+ vc.mutex = stripe->mutex;
+ vc.stripe = stripe;
+ rand_CacheKey(&vc.key);
+ vc.set_test_data("yay", 4);
+ vc.set_write_len(4);
+ vc.set_agg_len(stripe->round_to_approx_size(vc.write_len + vc.header_len + vc.frag_len + sizeof(Doc)));
+}
+
+// The doc of a head writer carries earliest_key, but openWriteCloseHeadDone registers its entry under first_key.
+void
+init_head_writer(FakeVC &vc, StripeSM *stripe)
+{
+ init_writer(vc, stripe);
+ vc.f.use_first_key = 1;
+ vc.earliest_key = vc.key;
+ rand_CacheKey(&vc.first_key);
+}
+
+CacheKey const *
+registered_key(FakeVC const &vc)
+{
+ if (vc.f.use_first_key) {
+ return &vc.first_key;
+ }
+ return &vc.key;
+}
+
+} // namespace
+
+class AggWriteErrorTest : public CacheInit
+{
+public:
+ int
+ cache_init_success_callback(int /* event ATS_UNUSED */, void * /* e ATS_UNUSED */) override
+ {
+ REQUIRE(CacheProcessor::IsCacheEnabled() == CacheInitState::INITIALIZED);
+ REQUIRE(gnstripes >= 1);
+
+ StripeSM *stripe = gstripes[0];
+ MUTEX_TRY_LOCK(lock, stripe->mutex, this_ethread());
+ if (!lock.is_locked()) {
+ CONT_SCHED_LOCK_RETRY(this);
+ return EVENT_DONE;
+ }
+
+ stripe->clear_dir();
+
+ std::array<FakeVC, 3> writers;
+ init_writer(writers[0], stripe); // case 1 - Sits at write_pos, the start of the buffer.
+ init_writer(writers[1], stripe); // case 2 - Sits past write_pos, so its offset includes its position in the buffer.
+ init_head_writer(writers[2], stripe); // case 3 - A head: registered under first_key, not its doc's key.
+ int agg_len = 0;
+ for (auto &vc : writers) {
+ REQUIRE(stripe->add_writer(&vc));
+ agg_len += vc.agg_len;
+ }
+
+ // Below the high water mark and with no sync writer, aggWrite fills the buffer without starting the disk write.
+ stripe->aggWrite(EVENT_CALL, nullptr);
+ REQUIRE(!stripe->is_io_in_progress());
+ REQUIRE(stripe->get_agg_buf_pos() == agg_len);
+
+ // A writer inserts its entry as soon as its doc is in the buffer, before the disk write completes.
Review Comment:
It makes sense, but I'm not changing this: the comment describes what the real writers do, which is why the test inserts the entries here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@trafficserver.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fix dir entry cleanup after a failed aggregation write [trafficserver]
Posted by "masaori335 (via GitHub)" <gi...@apache.org>.
masaori335 merged PR #13847:
URL: https://github.com/apache/trafficserver/pull/13847
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@trafficserver.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fix dir entry cleanup after a failed aggregation write [trafficserver]
Posted by "masaori335 (via GitHub)" <gi...@apache.org>.
masaori335 commented on code in PR #13847:
URL: https://github.com/apache/trafficserver/pull/13847#discussion_r4321594114
##########
src/iocore/cache/unit_tests/test_AggWriteDone.cc:
##########
@@ -0,0 +1,146 @@
+/** @file
+
+ Unit tests for StripeSM::aggWriteDone.
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "main.h"
+#include "test_doubles.h"
+
+#include "../P_CacheDir.h"
+#include "../P_CacheInternal.h"
+
+#include <array>
+#include <cerrno>
+
+// Required by main.h
+int cache_vols = 1;
+bool reuse_existing_cache = false;
+
+namespace
+{
+
+void
+init_writer(FakeVC &vc, StripeSM *stripe)
+{
+ // Sharing the stripe mutex lets aggWrite(EVENT_CALL) return the writer instead of scheduling a callback to it.
+ vc.mutex = stripe->mutex;
+ vc.stripe = stripe;
+ rand_CacheKey(&vc.key);
+ vc.set_test_data("yay", 4);
+ vc.set_write_len(4);
+ vc.set_agg_len(stripe->round_to_approx_size(vc.write_len + vc.header_len + vc.frag_len + sizeof(Doc)));
+}
+
+// The doc of a head writer carries earliest_key, but openWriteCloseHeadDone registers its entry under first_key.
+void
+init_head_writer(FakeVC &vc, StripeSM *stripe)
+{
+ init_writer(vc, stripe);
+ vc.f.use_first_key = 1;
+ vc.earliest_key = vc.key;
+ rand_CacheKey(&vc.first_key);
Review Comment:
Same as above.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@trafficserver.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Fix dir entry cleanup after a failed aggregation write [trafficserver]
Posted by "cmcfarlen (via GitHub)" <gi...@apache.org>.
cmcfarlen commented on PR #13847:
URL: https://github.com/apache/trafficserver/pull/13847#issuecomment-6508405914
Cherry-picked to the 10.2.x branch as v10.2.2 for the 10.2.2 release.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@trafficserver.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org