You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2026/07/21 12:41:46 UTC
(asterixdb) 01/07: [ASTERIXDB-3788][STO] flush .dic LAF pages in ascending page-id order
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0be4acd9589b45ed2a2a3530a80ff4bf2b67f13f
Author: Ritik Raj <ra...@gmail.com>
AuthorDate: Fri Jun 26 23:22:28 2026 +0530
[ASTERIXDB-3788][STO] flush .dic LAF pages in ascending page-id order
On cloud-backed deployments the columnar .dic (Look-Aside File) is
written through an append-only cloud writer whose ensurePosition
invariant requires pages to arrive in strictly ascending offset
(page-id) order. LAFWriter did not honor that:
- writeFullPage() flushed a LAF frame as soon as it filled, i.e. in
completion order, not page-id order; and
- endWriting() flushed the remaining frames in HashMap iteration
order.
During a large MERGE the bulkloader writes a leaf's column pages first
and its page-zero last, and multi-page column values add extra-page
entries that straddle LAF-page boundaries. So a higher LAF page could
fill and flush while a lower one was still incomplete. Harmless on the
random-access local file, but fatal on the append-only cloud copy:
AbstractCloudIOManager.ensurePosition throws "Misaligned positions" and
the merge thread halts the JVM.
Fix: hold a LAF frame back until every lower-id page has been flushed.
A nextPageToFlush cursor releases only the contiguous ascending prefix
of completed frames; endWriting flushes the remainder in ascending
page-id order. The local path is random-access and unaffected; only the
append-only cloud writer needs the ordering guarantee, and
ensurePosition stays intact as a corruption guard.
Adds a dedicated regression test, CloudStorageMergeTest, that runs only
cloud_storage/dic-merge-misalign: a COLUMN dataset with incompressible
columns inserted in batches, then COMPACT, forces a merged component
whose .dic spans multiple LAF pages. Without this change it halts the
JVM during COMPACT (mock S3, append-only cloud writer); with it the
merge completes and the count is correct.
To surface the bug with little data, the test uses its own config
(cc-cloud-storage-merge.conf.ftl: buffercache.pagesize=2KB,
memorycomponent.globalbudget=8MB, column.max.tuple.count=200) so a LAF
page holds few entries and inserts flush into many small components.
Kept as a separate test class + config so these aggressive storage
knobs do not perturb the shared CloudStorageTest suite.
Ext-ref: MB-72604
Co-Authored-By: Claude Opus 4.8 (1M context) <no...@anthropic.com>
Change-Id: I2a8a83cd1b57d3ceb1d9be381edd6589a0e138b9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21372
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <im...@apache.org>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Reviewed-by: Ritik Raj <ri...@couchbase.com>
---
.../test/cloud_storage/CloudStorageMergeTest.java | 135 +++++++++++++++++++++
.../test/resources/cc-cloud-storage-merge.conf.ftl | 78 ++++++++++++
.../dic-merge-misalign/mis.000.ddl.sqlpp | 37 ++++++
.../dic-merge-misalign/mis.001.update.sqlpp | 40 ++++++
.../dic-merge-misalign/mis.002.update.sqlpp | 40 ++++++
.../dic-merge-misalign/mis.003.update.sqlpp | 40 ++++++
.../dic-merge-misalign/mis.004.update.sqlpp | 29 +++++
.../dic-merge-misalign/mis.005.query.sqlpp | 25 ++++
.../dic-merge-misalign/mis.999.ddl.sqlpp | 20 +++
.../cloud_storage/dic-merge-misalign/mis.005.adm | 1 +
.../runtimets/testsuite_cloud_storage_merge.xml | 28 +++++
.../testsuite_cloud_storage_merge_only.xml | 23 ++++
.../storage/common/compression/file/LAFWriter.java | 57 ++++++---
13 files changed, 539 insertions(+), 14 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageMergeTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageMergeTest.java
new file mode 100644
index 0000000000..8db6906386
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/cloud_storage/CloudStorageMergeTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.cloud_storage;
+
+import static org.apache.asterix.api.common.LocalCloudUtilAdobeMock.createIfNotExists;
+import static org.apache.asterix.api.common.LocalCloudUtilAdobeMock.fillConfigTemplate;
+
+import java.net.URI;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.asterix.api.common.LocalCloudUtilAdobeMock;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.test.common.TestConstants;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.test.runtime.LangExecutionUtil;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.asterix.testframework.xml.Description;
+import org.apache.asterix.testframework.xml.TestCase;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.MethodSorters;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import com.adobe.testing.s3mock.testcontainers.S3MockContainer;
+
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
+
+/**
+ * Cloud-deployment regression test for ASTERIXDB-3788: a large columnar MERGE must flush the
+ * compressed .dic (LAF) pages to the append-only cloud writer in ascending page-id order.
+ *
+ * Runs ONLY the cloud_storage/dic-merge-misalign case, under its own config
+ * (cc-cloud-storage-merge.conf.ftl: small buffercache.pagesize + low globalbudget +
+ * small column.max.tuple.count) so a merged component's .dic spans multiple LAF pages
+ * with little data. Kept separate from {@link CloudStorageTest} so those aggressive
+ * storage knobs do not perturb the shared cloud-storage suite.
+ */
+@RunWith(Parameterized.class)
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class CloudStorageMergeTest {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final TestCaseContext tcCtx;
+ public static final String SUITE_TESTS = "testsuite_cloud_storage_merge.xml";
+ public static final String ONLY_TESTS = "testsuite_cloud_storage_merge_only.xml";
+ public static final String CONFIG_FILE_TEMPLATE = "src/test/resources/cc-cloud-storage-merge.conf.ftl";
+ public static final String CONFIG_FILE = "target/cc-cloud-storage-merge.conf";
+ public static final String DELTA_RESULT_PATH = "results_cloud";
+ public static final String EXCLUDED_TESTS = "MP";
+
+ public static final String PLAYGROUND_CONTAINER = "playground";
+ public static final String MOCK_SERVER_REGION = "us-west-2";
+ public static final String MOCK_SERVER_HOSTNAME_FRAGMENT = "http://127.0.0.1:";
+
+ public CloudStorageMergeTest(TestCaseContext tcCtx) {
+ this.tcCtx = tcCtx;
+ }
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ TestExecutor testExecutor = new TestExecutor(DELTA_RESULT_PATH);
+ setupEnv(testExecutor);
+ }
+
+ public static void setupEnv(TestExecutor testExecutor) throws Exception {
+ S3MockContainer s3Mock = LocalCloudUtilAdobeMock.startS3CloudEnvironment(true);
+ fillConfigTemplate(MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort(), CONFIG_FILE_TEMPLATE,
+ CONFIG_FILE);
+ System.setProperty(TestConstants.S3_SERVICE_ENDPOINT_KEY,
+ MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort());
+ testExecutor.executorId = "cloud";
+ testExecutor.stripSubstring = "//DB:";
+ LangExecutionUtil.setUp(CONFIG_FILE, testExecutor);
+ System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, CONFIG_FILE);
+
+ // create the playground bucket and leave it empty, just for external collection-based tests
+ S3ClientBuilder builder = S3Client.builder();
+ URI endpoint = URI.create(MOCK_SERVER_HOSTNAME_FRAGMENT + s3Mock.getHttpServerPort()); // endpoint pointing to S3 mock server
+ builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
+ .endpointOverride(endpoint);
+ S3Client client = builder.build();
+ createIfNotExists(PLAYGROUND_CONTAINER, client);
+ client.close();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ LangExecutionUtil.tearDown();
+ LocalCloudUtilAdobeMock.shutdownSilently();
+ }
+
+ @Parameters(name = "CloudStorageMergeTest {index}: {0}")
+ public static Collection<Object[]> tests() throws Exception {
+ return LangExecutionUtil.tests(ONLY_TESTS, SUITE_TESTS);
+ }
+
+ @Test
+ public void test() throws Exception {
+ List<TestCase.CompilationUnit> cu = tcCtx.getTestCase().getCompilationUnit();
+ Assume.assumeTrue(cu.size() > 1 || !EXCLUDED_TESTS.equals(getText(cu.get(0).getDescription())));
+ LangExecutionUtil.test(tcCtx);
+ }
+
+ private static String getText(Description description) {
+ return description == null ? "" : description.getValue();
+ }
+}
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-merge.conf.ftl b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-merge.conf.ftl
new file mode 100644
index 0000000000..1bd6c5d37f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-merge.conf.ftl
@@ -0,0 +1,78 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied. See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[nc/asterix_nc1]
+txn.log.dir=target/tmp/asterix_nc1/txnlog
+core.dump.dir=target/tmp/asterix_nc1/coredump
+iodevices=target/tmp/asterix_nc1/iodevice1
+iodevices=../asterix-server/target/tmp/asterix_nc1/iodevice2
+nc.api.port=19004
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5006
+
+[nc/asterix_nc2]
+ncservice.port=9091
+txn.log.dir=target/tmp/asterix_nc2/txnlog
+core.dump.dir=target/tmp/asterix_nc2/coredump
+iodevices=target/tmp/asterix_nc2/iodevice1,../asterix-server/target/tmp/asterix_nc2/iodevice2
+nc.api.port=19005
+#jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5007
+
+[nc]
+credential.file=src/test/resources/security/passwd
+python.cmd.autolocate=true
+python.env=FOO=BAR=BAZ,BAR=BAZ
+address=127.0.0.1
+command=asterixnc
+app.class=org.apache.asterix.hyracks.bootstrap.NCApplication
+jvm.args=-Xmx4096m --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
+storage.buffercache.size=128MB
+storage.memorycomponent.globalbudget=8MB
+storage.max.columns.in.zeroth.segment=800
+storage.page.zero.writer=default
+
+[cc]
+address = 127.0.0.1
+app.class=org.apache.asterix.hyracks.bootstrap.CCApplication
+heartbeat.period=2000
+heartbeat.max.misses=25
+credential.file=src/test/resources/security/passwd
+
+[common]
+log.dir = logs/
+log.level = INFO
+compiler.framesize=32KB
+compiler.sortmemory=320KB
+compiler.groupmemory=160KB
+compiler.joinmemory=256KB
+compiler.textsearchmemory=160KB
+compiler.windowmemory=192KB
+compiler.ordered.fields=false
+compiler.internal.sanitycheck=true
+messaging.frame.size=4096
+messaging.frame.count=512
+cloud.deployment=true
+storage.buffercache.pagesize=2KB
+storage.column.max.tuple.count=200
+storage.partitioning=static
+cloud.storage.scheme=s3
+cloud.storage.bucket=cloud-storage-container
+cloud.storage.region=us-west-2
+cloud.storage.endpoint=${cloudUrl}
+cloud.storage.anonymous.auth=true
+cloud.storage.cache.policy=selective
+cloud.max.write.requests.per.second=2000
+cloud.max.read.requests.per.second=4000
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.000.ddl.sqlpp
new file mode 100644
index 0000000000..03d570ff91
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.000.ddl.sqlpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- CBSE-23088-4 repro: out-of-order .dic LAF page flush vs append-only cloud writer.
+-- COLUMN-format dataset with FAT nested-array columns (mimics scenario_execution_detail
+-- ExecutionDetail / ScenarioResult). The fat columns make a single column span many
+-- data pages so the component's .dic spans more than one LAF page during the merge.
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+ id: bigint
+};
+
+CREATE DATASET ColumnDataset(OpenType)
+PRIMARY KEY id WITH {
+ "storage-format": {"format" : "column"}
+};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.001.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.001.update.sqlpp
new file mode 100644
index 0000000000..b8de7e6253
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.001.update.sqlpp
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- Batch 1. HIGH-CARDINALITY incompressible columns: every field is a distinct pseudo-random
+-- int per (gen,e) so dictionary/delta encoding cannot collapse the column -> one column
+-- spans many data pages -> the component .dic spans many LAF pages during MERGE.
+
+USE test;
+
+INSERT INTO ColumnDataset
+SELECT VALUE {
+ "id": gen,
+ "scenarioResult": (SELECT VALUE {
+ "k": (gen*2654435761 + e*40503) % 2147483647,
+ "v": (e*2654435761 + gen*40503) % 2147483647,
+ "w": (gen*e*2246822519 + e*7919) % 2147483647
+ } FROM range(1, 250) AS e),
+ "executionDetail": (SELECT VALUE {
+ "step": (gen*1000003 + e*97) % 2147483647,
+ "ts": (gen*2147483629 + e*65537) % 2147483647,
+ "note": (e*2654435761 + gen*99991) % 2147483647
+ } FROM range(1, 250) AS e)
+}
+FROM range(1, 6000) AS gen;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.002.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.002.update.sqlpp
new file mode 100644
index 0000000000..00ba2d81bf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.002.update.sqlpp
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- Batch 2. HIGH-CARDINALITY incompressible columns: every field is a distinct pseudo-random
+-- int per (gen,e) so dictionary/delta encoding cannot collapse the column -> one column
+-- spans many data pages -> the component .dic spans many LAF pages during MERGE.
+
+USE test;
+
+INSERT INTO ColumnDataset
+SELECT VALUE {
+ "id": gen,
+ "scenarioResult": (SELECT VALUE {
+ "k": (gen*2654435761 + e*40503) % 2147483647,
+ "v": (e*2654435761 + gen*40503) % 2147483647,
+ "w": (gen*e*2246822519 + e*7919) % 2147483647
+ } FROM range(1, 250) AS e),
+ "executionDetail": (SELECT VALUE {
+ "step": (gen*1000003 + e*97) % 2147483647,
+ "ts": (gen*2147483629 + e*65537) % 2147483647,
+ "note": (e*2654435761 + gen*99991) % 2147483647
+ } FROM range(1, 250) AS e)
+}
+FROM range(2500001, 2506000) AS gen;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.003.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.003.update.sqlpp
new file mode 100644
index 0000000000..b4b4fc969f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.003.update.sqlpp
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- Batch 3. HIGH-CARDINALITY incompressible columns: every field is a distinct pseudo-random
+-- int per (gen,e) so dictionary/delta encoding cannot collapse the column -> one column
+-- spans many data pages -> the component .dic spans many LAF pages during MERGE.
+
+USE test;
+
+INSERT INTO ColumnDataset
+SELECT VALUE {
+ "id": gen,
+ "scenarioResult": (SELECT VALUE {
+ "k": (gen*2654435761 + e*40503) % 2147483647,
+ "v": (e*2654435761 + gen*40503) % 2147483647,
+ "w": (gen*e*2246822519 + e*7919) % 2147483647
+ } FROM range(1, 250) AS e),
+ "executionDetail": (SELECT VALUE {
+ "step": (gen*1000003 + e*97) % 2147483647,
+ "ts": (gen*2147483629 + e*65537) % 2147483647,
+ "note": (e*2654435761 + gen*99991) % 2147483647
+ } FROM range(1, 250) AS e)
+}
+FROM range(5000001, 5006000) AS gen;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.004.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.004.update.sqlpp
new file mode 100644
index 0000000000..970f36da46
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.004.update.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- THE TRIGGER. COMPACT folds the flushed components into one large component via the
+-- MERGE path (ColumnBTreeBulkloader.persist -> writeColumnsPages). The merged component's
+-- .dic spans multiple LAF pages (fat columns -> many data pages); page-zero is written
+-- LAST (low LAF id) so a higher LAF page flushes first -> out-of-order append to the
+-- append-only cloud .dic writer -> AbstractCloudIOManager.ensurePosition throws
+-- "Misaligned positions" -> MERGE thread halts the JVM.
+
+USE test;
+
+COMPACT DATASET ColumnDataset;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.005.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.005.query.sqlpp
new file mode 100644
index 0000000000..60fb41dd42
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.005.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+-- Only reached if the merge did NOT crash. If the bug reproduced, the JVM halted during
+-- step 004 and this step never runs (the whole test process dies = repro confirmed).
+
+USE test;
+
+SELECT VALUE COUNT(*) FROM ColumnDataset;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.999.ddl.sqlpp
new file mode 100644
index 0000000000..20dc6fde5e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cloud_storage/dic-merge-misalign/mis.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/dic-merge-misalign/mis.005.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/dic-merge-misalign/mis.005.adm
new file mode 100644
index 0000000000..8f31360b2d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/cloud_storage/dic-merge-misalign/mis.005.adm
@@ -0,0 +1 @@
+18000
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage_merge.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage_merge.xml
new file mode 100644
index 0000000000..c6820fd3df
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage_merge.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
+ <test-group name="cloud_storage">
+ <test-case FilePath="cloud_storage">
+ <compilation-unit name="dic-merge-misalign">
+ <output-dir compare="Text">dic-merge-misalign</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+</test-suite>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage_merge_only.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage_merge_only.xml
new file mode 100644
index 0000000000..a28be6cfa3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_cloud_storage_merge_only.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
+ <test-group name="cloud_storage">
+ </test-group>
+</test-suite>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/compression/file/LAFWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/compression/file/LAFWriter.java
index 1ed462b333..603b674f4b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/compression/file/LAFWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/compression/file/LAFWriter.java
@@ -44,6 +44,13 @@ import org.apache.hyracks.util.annotations.NotThreadSafe;
*/
@NotThreadSafe
class LAFWriter implements ICompressedPageWriter {
+ // Lowest LAF page id not yet flushed. A frame is released to the (dual local+cloud) page
+ // writer only after every lower-id page has been flushed, so the append-only cloud .dic
+ // writer always receives pages in ascending offset order (otherwise a higher LAF page can
+ // flush while a lower one is still incomplete -> AbstractCloudIOManager.ensurePosition
+ // "Misaligned positions" -> merge halts the JVM). The local file is random-access and
+ // indifferent to flush order; only the cloud copy needs this guarantee. See ASTERIXDB-3788.
+ private int nextPageToFlush;
private final CompressedFileManager compressedFileManager;
private final IBufferCache bufferCache;
private final IFIFOPageWriter pageWriter;
@@ -73,6 +80,7 @@ class LAFWriter implements ICompressedPageWriter {
totalNumOfPages = 0;
maxPageId = -1;
currentPageId = -1;
+ nextPageToFlush = 0;
}
/* ************************************
@@ -140,8 +148,14 @@ class LAFWriter implements ICompressedPageWriter {
*/
lastPage.setEOF();
}
- for (Entry<Integer, LAFFrame> entry : cachedFrames.entrySet()) {
- pageWriter.write(entry.getValue().cPage);
+ // Flush remaining frames in ASCENDING page-id order. The append-only cloud .dic writer
+ // requires ascending offsets; HashMap iteration order does not guarantee that and would
+ // trip AbstractCloudIOManager.ensurePosition ("Misaligned positions"). ASTERIXDB-3788.
+ for (int pageId = nextPageToFlush; pageId <= maxPageId; pageId++) {
+ final LAFFrame frame = cachedFrames.remove(pageId);
+ if (frame != null) {
+ flushFrame(pageId, frame);
+ }
}
//Signal the compressedFileManager to change its state
@@ -169,7 +183,7 @@ class LAFWriter implements ICompressedPageWriter {
lastOffset += size;
totalNumOfPages++;
- writeFullPage();
+ flushReadyPages();
return pageOffset;
}
@@ -194,20 +208,35 @@ class LAFWriter implements ICompressedPageWriter {
return frame;
}
- private void writeFullPage() throws HyracksDataException {
- if (currentFrame.isFull()) {
- //The LAF page is filled. We do not need to keep it.
- //Write it to the file and remove it from the cachedFrames map
- pageWriter.write(currentFrame.cPage);
- //Recycle the frame
- final LAFFrame frame = cachedFrames.remove(currentPageId);
- frame.setCachedPage(null);
- recycledFrames.add(frame);
- currentFrame = null;
- currentPageId = -1;
+ /**
+ * Flush the longest contiguous run of FULL LAF pages starting at {@link #nextPageToFlush}.
+ * A higher LAF page that fills before a lower one is held back (it stays in
+ * {@code cachedFrames}) until every lower-id page has been flushed, so the append-only
+ * cloud .dic writer always receives pages in ascending offset order. The local file is
+ * random-access and indifferent to order; only the cloud copy needs this guarantee.
+ */
+ private void flushReadyPages() throws HyracksDataException {
+ LAFFrame frame;
+ while ((frame = cachedFrames.get(nextPageToFlush)) != null && frame.isFull()) {
+ final int pageId = nextPageToFlush;
+ cachedFrames.remove(pageId);
+ flushFrame(pageId, frame);
+ if (currentPageId == pageId) {
+ currentFrame = null;
+ currentPageId = -1;
+ }
}
}
+ /** Write one LAF page to the (dual) page writer and recycle its frame. Advances
+ * {@link #nextPageToFlush}; callers MUST invoke this in ascending page-id order. */
+ private void flushFrame(int pageId, LAFFrame frame) throws HyracksDataException {
+ pageWriter.write(frame.cPage);
+ frame.setCachedPage(null);
+ recycledFrames.add(frame);
+ nextPageToFlush = pageId + 1;
+ }
+
private int getLAFEntryPageId(int compressedPageId) {
return compressedPageId * ENTRY_LENGTH / bufferCache.getPageSize();
}