You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2017/11/09 18:07:38 UTC
asterixdb git commit: [ASTERIXDB-2130][TX] Prevent Dropping Index
Pending Txn Completion
Repository: asterixdb
Updated Branches:
refs/heads/master 7c53fcf40 -> 7f00136dc
[ASTERIXDB-2130][TX] Prevent Dropping Index Pending Txn Completion
- user model changes: no
- storage format changes: no
- interface changes: yes
Added resource id to ITransactionOperationTracker
before/after txn operations.
Details:
- Currently, an index could be evicted/dropped while a transaction
waiting for its completion (commit/rollback). This change prevents
that by incrementing the reference counter of the indexes registered
in the transaction.
- Add test case.
Change-Id: If6a938cbb5a9c3b7f5cc59505c07ae45b3425223
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2132
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/7f00136d
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/7f00136d
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/7f00136d
Branch: refs/heads/master
Commit: 7f00136dc098c6e7956de6843323c5affd887723
Parents: 7c53fcf
Author: Murtadha Hubail <mh...@apache.org>
Authored: Tue Nov 7 18:35:58 2017 +0300
Committer: Murtadha Hubail <mh...@apache.org>
Committed: Thu Nov 9 10:06:39 2017 -0800
----------------------------------------------------------------------
.../test/dataflow/ComponentRollbackTest.java | 1 +
.../IndexDropOperatorNodePushableTest.java | 59 ++++++++++++++++++++
.../common/context/BaseOperationTracker.java | 10 +++-
.../context/ITransactionOperationTracker.java | 10 +++-
.../service/transaction/TransactionContext.java | 4 +-
5 files changed, 78 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7f00136d/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
index ac5e606..6bff50d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java
@@ -264,6 +264,7 @@ public class ComponentRollbackTest {
tupleAppender.write(insertOp, true);
}
insertOp.close();
+ nc.getTransactionManager().completedTransaction(txnCtx, DatasetId.NULL, -1, true);
searchAndAssertCount(nc, ctx, dataset, storageManager, TOTAL_NUM_OF_RECORDS);
// rollback the last disk component
lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7f00136d/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
index 2bac49e..7ced87d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
@@ -30,17 +30,28 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.asterix.app.bootstrap.TestNodeController;
import org.apache.asterix.common.config.DatasetConfig;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.file.StorageComponentProvider;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities;
+import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
+import org.apache.asterix.metadata.utils.SplitsAndConstraintsUtil;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.test.common.TestHelper;
+import org.apache.asterix.test.runtime.ExecutionTestUtil;
+import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
+import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorNodePushable;
@@ -64,6 +75,7 @@ public class IndexDropOperatorNodePushableTest {
private static final String DATA_TYPE_NAME = "DUMMY";
private static final String NODE_GROUP_NAME = "DEFAULT";
private final AtomicBoolean dropFailed = new AtomicBoolean(false);
+ private final TestExecutor testExecutor = new TestExecutor();
@Before
public void setUp() throws Exception {
@@ -77,6 +89,12 @@ public class IndexDropOperatorNodePushableTest {
TestHelper.deleteExistingInstanceFiles();
}
+ /**
+ * Tests dropping a dataset using different
+ * drop options
+ *
+ * @throws Exception
+ */
@Test
public void dropOptionsTest() throws Exception {
TestNodeController nc = new TestNodeController(null, false);
@@ -107,6 +125,47 @@ public class IndexDropOperatorNodePushableTest {
}
}
+ /**
+ * Tests dropping an index whose dataset has no active
+ * operations
+ *
+ * @throws Exception
+ */
+ @Test
+ public void dropIndexInUseTest() throws Exception {
+ TestNodeController nc = new TestNodeController(null, false);
+ try {
+ nc.init();
+ String datasetName = "ds";
+ String indexName = "fooIdx";
+ // create dataset and index
+ final TestCaseContext.OutputFormat format = TestCaseContext.OutputFormat.CLEAN_JSON;
+ testExecutor.executeSqlppUpdateOrDdl("CREATE TYPE KeyType AS { id: int, foo: int };", format);
+ testExecutor.executeSqlppUpdateOrDdl("CREATE DATASET " + datasetName + "(KeyType) PRIMARY KEY id;", format);
+ testExecutor.executeSqlppUpdateOrDdl("CREATE INDEX " + indexName + " on " + datasetName + "(foo)", format);
+ final MetadataTransactionContext mdTxn = MetadataManager.INSTANCE.beginTransaction();
+ ICcApplicationContext appCtx =
+ (ICcApplicationContext) ExecutionTestUtil.integrationUtil.getClusterControllerService()
+ .getApplicationContext();
+ MetadataProvider metadataProver = new MetadataProvider(appCtx, null);
+ metadataProver.setMetadataTxnContext(mdTxn);
+ final String defaultDv = MetadataBuiltinEntities.DEFAULT_DATAVERSE.getDataverseName();
+ final Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxn, defaultDv, datasetName);
+ MetadataManager.INSTANCE.commitTransaction(mdTxn);
+ FileSplit[] splits = SplitsAndConstraintsUtil
+ .getIndexSplits(appCtx.getClusterStateManager(), dataset, indexName, Arrays.asList("asterix_nc1"));
+ final ConstantFileSplitProvider constantFileSplitProvider =
+ new ConstantFileSplitProvider(Arrays.copyOfRange(splits, 0, 1));
+ IndexDataflowHelperFactory helperFactory =
+ new IndexDataflowHelperFactory(nc.getStorageManager(), constantFileSplitProvider);
+ IHyracksTaskContext ctx = nc.createTestContext(true);
+ IIndexDataflowHelper dataflowHelper = helperFactory.create(ctx.getJobletContext().getServiceContext(), 0);
+ dropInUse(ctx, helperFactory, dataflowHelper);
+ } finally {
+ nc.deInit();
+ }
+ }
+
private void dropInUse(IHyracksTaskContext ctx, IndexDataflowHelperFactory helperFactory,
IIndexDataflowHelper dataflowHelper) throws Exception {
dropFailed.set(false);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7f00136d/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
index 2068192..9f57981 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
@@ -63,12 +63,18 @@ public class BaseOperationTracker implements ITransactionOperationTracker {
}
@Override
- public void beforeTransaction() {
+ public void beforeTransaction(long resourceId) {
+ /*
+ * Increment dataset and index ref count to prevent them
+ * from being evicted/dropped until the transaction completes
+ */
dsInfo.touch();
+ dsInfo.getIndexes().get(resourceId).touch();
}
@Override
- public void afterTransaction() {
+ public void afterTransaction(long resourceId) {
dsInfo.untouch();
+ dsInfo.getIndexes().get(resourceId).untouch();
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7f00136d/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
index e6aeec5..301801b 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
@@ -25,12 +25,18 @@ public interface ITransactionOperationTracker extends ILSMOperationTracker {
/**
* Called before a transaction performs any operations on
* {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex}
+ * with resource id {@code resourceId}
+ *
+ * @param resourceId
*/
- void beforeTransaction();
+ void beforeTransaction(long resourceId);
/**
* Called after a transaction completes its operations on
* {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex}
+ * with resource id {@code resourceId}
+ *
+ * @param resourceId
*/
- void afterTransaction();
+ void afterTransaction(long resourceId);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7f00136d/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
index 4846b6e..3159e6b 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
@@ -122,7 +122,7 @@ public class TransactionContext implements ITransactionContext {
final ITransactionOperationTracker txnOpTracker =
(ITransactionOperationTracker) index.getOperationTracker();
indexMap.put(resourceId, txnOpTracker);
- txnOpTracker.beforeTransaction();
+ txnOpTracker.beforeTransaction(resourceId);
}
}
}
@@ -258,7 +258,7 @@ public class TransactionContext implements ITransactionContext {
}
} finally {
synchronized (indexMap) {
- indexMap.values().forEach(ITransactionOperationTracker::afterTransaction);
+ indexMap.forEach((resource, opTracker) -> opTracker.afterTransaction(resource));
}
}
}