You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org> on 2018/03/10 12:02:17 UTC
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Murtadha Hubail has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2471
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
[ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Pass correct merging components in case of
full merge.
- Add full merge test case.
Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
---
A asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/LsmIndexLifecycleTest.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
2 files changed, 74 insertions(+), 1 deletion(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/71/2471/1
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/LsmIndexLifecycleTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/LsmIndexLifecycleTest.java
new file mode 100644
index 0000000..15f1730
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/LsmIndexLifecycleTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.storage;
+
+import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
+import org.apache.asterix.common.TestDataUtil;
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.context.DatasetInfo;
+import org.apache.asterix.metadata.api.IMetadataIndex;
+import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class LsmIndexLifecycleTest {
+
+ protected static final String TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
+ private static final TestExecutor testExecutor = new TestExecutor();
+ private static final AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil();
+
+ @Before
+ public void setUp() throws Exception {
+ System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, TEST_CONFIG_FILE_NAME);
+ integrationUtil.init(true, TEST_CONFIG_FILE_NAME);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ integrationUtil.deinit(true);
+ }
+
+ @Test
+ public void fullMergeTest() throws Exception {
+ String datasetName = "ds";
+ TestDataUtil.createIdOnlyDataset(datasetName);
+ INcApplicationContext appCtx = (INcApplicationContext) (integrationUtil.ncs[0].getApplicationContext());
+ IDatasetLifecycleManager dlcm = appCtx.getDatasetLifecycleManager();
+ IMetadataIndex dsIdx = MetadataPrimaryIndexes.DATASET_DATASET;
+ DatasetInfo datasetInfo = dlcm.getDatasetInfo(dsIdx.getDatasetId().getId());
+ // flush to ensure multiple disk components
+ dlcm.flushAllDatasets();
+ datasetInfo.waitForIO();
+ AbstractLSMIndex index = (AbstractLSMIndex) dlcm.getIndex(dsIdx.getDatasetId().getId(), dsIdx.getResourceId());
+ Assert.assertTrue(index.getDiskComponents().size() > 1);
+ // trigger full merge and ensure we have a single disk component when merge completes
+ testExecutor.executeSqlppUpdateOrDdl("COMPACT DATASET Metadata.`Dataset`;",
+ TestCaseContext.OutputFormat.CLEAN_JSON);
+ datasetInfo.waitForIO();
+ Assert.assertTrue(index.getDiskComponents().size() == 1);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index bd91094..af8c702 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -369,7 +369,7 @@
AbstractLSMIndexOperationContext opCtx = createOpContext(NoOpIndexAccessParameters.INSTANCE);
opCtx.setOperation(ctx.getOperation());
opCtx.getComponentHolder().addAll(mergingComponents);
- opCtx.getComponentsToBeMerged().addAll(ctx.getComponentsToBeMerged());
+ mergingComponents.stream().map(ILSMDiskComponent.class::cast).forEach(opCtx.getComponentsToBeMerged()::add);
ILSMDiskComponent firstComponent = (ILSMDiskComponent) mergingComponents.get(0);
ILSMDiskComponent lastComponent = (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1);
LSMComponentFileReferences mergeFileRefs = getMergeFileReferences(firstComponent, lastComponent);
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-verify-asterix-app/3479/ (4/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/5905/
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Luo Chen (Code Review)" <do...@asterixdb.incubator.apache.org>.
Luo Chen has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1: Code-Review+2
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Luo Chen <cl...@uci.edu>
Gerrit-Reviewer: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Luo Chen (Code Review)" <do...@asterixdb.incubator.apache.org>.
Luo Chen has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
(1 comment)
https://asterix-gerrit.ics.uci.edu/#/c/2471/1/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
File hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java:
PS1, Line 372: mergingComponents.stream().map(ILSMDiskComponent.class::cast).forEach(opCtx.getComponentsToBeMerged()::add);
why we need a class cast here?
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Luo Chen <cl...@uci.edu>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-source-format/2966/ (10/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/hyracks-gerrit/2918/ (7/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-ensure-ancestor/1013/ (5/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Anon. E. Moose (Code Review)" <do...@asterixdb.incubator.apache.org>.
Anon. E. Moose #1000171 has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Analytics Compatibility Compilation Successful
https://goo.gl/Bb5rSZ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/8463/ (12/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Anon. E. Moose (Code Review)" <do...@asterixdb.incubator.apache.org>.
Anon. E. Moose #1000171 has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1: Contrib+1
Analytics Compatibility Tests Successful
https://goo.gl/DBxVmA : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
WARNING: THIS CHANGE CONTAINS CROSS-PRODUCT CHANGES IN:
* asterixdb
* hyracks-fullstack
PLEASE REVIEW CAREFULLY AND LOOK FOR API CHANGES!
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-verify-no-installer-app/3455/ (2/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
BAD Compatibility Tests Started https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/2868/
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-source-assemblies/3223/ (3/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org>.
Murtadha Hubail has submitted this change and it was merged.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
[ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Pass correct merging components in case of
full merge.
- Add full merge test case.
Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2471
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Luo Chen <cl...@uci.edu>
---
A asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/LsmIndexLifecycleTest.java
M hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
2 files changed, 74 insertions(+), 1 deletion(-)
Approvals:
Luo Chen: Looks good to me, approved
Anon. E. Moose #1000171:
Jenkins: Verified; No violations found; ; Verified
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/LsmIndexLifecycleTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/LsmIndexLifecycleTest.java
new file mode 100644
index 0000000..15f1730
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/LsmIndexLifecycleTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.storage;
+
+import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
+import org.apache.asterix.common.TestDataUtil;
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.context.DatasetInfo;
+import org.apache.asterix.metadata.api.IMetadataIndex;
+import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class LsmIndexLifecycleTest {
+
+ protected static final String TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
+ private static final TestExecutor testExecutor = new TestExecutor();
+ private static final AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil();
+
+ @Before
+ public void setUp() throws Exception {
+ System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, TEST_CONFIG_FILE_NAME);
+ integrationUtil.init(true, TEST_CONFIG_FILE_NAME);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ integrationUtil.deinit(true);
+ }
+
+ @Test
+ public void fullMergeTest() throws Exception {
+ String datasetName = "ds";
+ TestDataUtil.createIdOnlyDataset(datasetName);
+ INcApplicationContext appCtx = (INcApplicationContext) (integrationUtil.ncs[0].getApplicationContext());
+ IDatasetLifecycleManager dlcm = appCtx.getDatasetLifecycleManager();
+ IMetadataIndex dsIdx = MetadataPrimaryIndexes.DATASET_DATASET;
+ DatasetInfo datasetInfo = dlcm.getDatasetInfo(dsIdx.getDatasetId().getId());
+ // flush to ensure multiple disk components
+ dlcm.flushAllDatasets();
+ datasetInfo.waitForIO();
+ AbstractLSMIndex index = (AbstractLSMIndex) dlcm.getIndex(dsIdx.getDatasetId().getId(), dsIdx.getResourceId());
+ Assert.assertTrue(index.getDiskComponents().size() > 1);
+ // trigger full merge and ensure we have a single disk component when merge completes
+ testExecutor.executeSqlppUpdateOrDdl("COMPACT DATASET Metadata.`Dataset`;",
+ TestCaseContext.OutputFormat.CLEAN_JSON);
+ datasetInfo.waitForIO();
+ Assert.assertTrue(index.getDiskComponents().size() == 1);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index bd91094..af8c702 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -369,7 +369,7 @@
AbstractLSMIndexOperationContext opCtx = createOpContext(NoOpIndexAccessParameters.INSTANCE);
opCtx.setOperation(ctx.getOperation());
opCtx.getComponentHolder().addAll(mergingComponents);
- opCtx.getComponentsToBeMerged().addAll(ctx.getComponentsToBeMerged());
+ mergingComponents.stream().map(ILSMDiskComponent.class::cast).forEach(opCtx.getComponentsToBeMerged()::add);
ILSMDiskComponent firstComponent = (ILSMDiskComponent) mergingComponents.get(0);
ILSMDiskComponent lastComponent = (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1);
LSMComponentFileReferences mergeFileRefs = getMergeFileReferences(firstComponent, lastComponent);
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Luo Chen <cl...@uci.edu>
Gerrit-Reviewer: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-asterix-app/3727/ (8/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-sonar/6956/ (6/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Murtadha Hubail (Code Review)" <do...@asterixdb.incubator.apache.org>.
Murtadha Hubail has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
(1 comment)
https://asterix-gerrit.ics.uci.edu/#/c/2471/1/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
File hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java:
PS1, Line 372: mergingComponents.stream().map(ILSMDiskComponent.class::cast).forEach(opCtx.getComponentsToBeMerged()::add);
> why we need a class cast here?
ComponentsToBeMerged is a list of ILSMDiskComponent
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Luo Chen <cl...@uci.edu>
Gerrit-Reviewer: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: abdullah alamoudi <ba...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1: Integration-Tests+1
Integration Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/5905/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-verify-storage/3537/ (9/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1: Contrib+1
BAD Compatibility Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/2868/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-asterix-app-sql-execution/3005/ (1/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: [ASTERIXDB-2316][STO] Fix Merging Components For Full Merge
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-cancellation-test/2993/ (11/12)
--
To view, visit https://asterix-gerrit.ics.uci.edu/2471
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: I17890a1f85f569e0d3dc5eba046f61c91571c9c1
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mh...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No