You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2015/05/20 03:25:50 UTC
[09/10] incubator-asterixdb git commit: Temp dataset support: 1. DDLs
for creating a temporary dataset 2. Garbage collection for temporary dataset
3. Reading, inserting,
and deleting data from (to) a temporary dataset is locking-free (except
metadata loc
Temp dataset support:
1. DDLs for creating a temporary dataset
2. Garbage collection for temporary dataset
3. Reading, inserting, and deleting data from (to) a temporary dataset is locking-free (except metadata locks) and logging-free (except flush/merge/job-commit logs).
Change-Id: Id0960acd1b4dfbfa7685dba227634572ca50b126
Reviewed-on: https://asterix-gerrit.ics.uci.edu/241
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Young-Seok Kim <ki...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/ccd67fe8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/ccd67fe8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/ccd67fe8
Branch: refs/heads/master
Commit: ccd67fe832b1396a9d637eb34212e68f5d39ff19
Parents: f61bdb8
Author: buyingyi <bu...@gmail.com>
Authored: Tue May 19 11:53:39 2015 -0700
Committer: Ian Maxon <im...@uci.edu>
Committed: Tue May 19 17:24:27 2015 -0700
----------------------------------------------------------------------
.../operators/physical/CommitPOperator.java | 14 +-
.../operators/physical/CommitRuntime.java | 43 ++-
.../physical/CommitRuntimeFactory.java | 8 +-
.../physical/InvertedIndexPOperator.java | 15 +-
.../rules/ReplaceSinkOpWithCommitOpRule.java | 3 +-
.../ics/asterix/api/common/APIFramework.java | 18 +-
.../asterix/aql/translator/AqlTranslator.java | 24 +-
.../uci/ics/asterix/file/DatasetOperations.java | 21 +-
.../file/ExternalIndexingOperations.java | 32 +-
.../uci/ics/asterix/file/IndexOperations.java | 7 +-
.../file/SecondaryBTreeOperationsHelper.java | 16 +-
.../file/SecondaryIndexOperationsHelper.java | 24 +-
.../SecondaryInvertedIndexOperationsHelper.java | 6 +-
.../file/SecondaryRTreeOperationsHelper.java | 17 +-
.../bootstrap/NCApplicationEntryPoint.java | 27 +-
.../basic/temp_dataset/temp_dataset.1.ddl.aql | 34 +++
.../temp_dataset/temp_dataset.2.update.aql | 6 +
.../basic/temp_dataset/temp_dataset.3.query.aql | 11 +
.../basic/temp_dataset/temp_dataset.1.adm | 1 +
.../src/test/resources/metadata/testsuite.xml | 5 +
...ete-from-loaded-dataset-with-index.1.ddl.aql | 34 +++
...-from-loaded-dataset-with-index.2.update.aql | 13 +
...ete-from-loaded-dataset-with-index.3.ddl.aql | 12 +
...-from-loaded-dataset-with-index.4.update.aql | 10 +
...e-from-loaded-dataset-with-index.5.query.aql | 13 +
.../delete-from-loaded-dataset.1.ddl.aql | 34 +++
.../delete-from-loaded-dataset.2.update.aql | 15 +
.../delete-from-loaded-dataset.3.query.aql | 12 +
.../drop-empty-secondary-indexes.1.ddl.aql | 57 ++++
.../drop-empty-secondary-indexes.2.update.aql | 7 +
.../drop-empty-secondary-indexes.3.query.aql | 10 +
.../drop-index/drop-index.1.ddl.aql | 33 ++
.../drop-index/drop-index.2.update.aql | 14 +
.../drop-index/drop-index.3.ddl.aql | 17 ++
.../drop-index/drop-index.4.query.aql | 12 +
.../empty-load-with-index.1.ddl.aql | 34 +++
.../empty-load-with-index.2.update.aql | 15 +
.../empty-load-with-index.3.query.aql | 12 +
...insert-and-scan-dataset-with-index.1.ddl.aql | 21 ++
...ert-and-scan-dataset-with-index.2.update.aql | 24 ++
...sert-and-scan-dataset-with-index.3.query.aql | 13 +
.../insert-and-scan-dataset.1.ddl.aql | 18 ++
.../insert-and-scan-dataset.2.update.aql | 21 ++
.../insert-and-scan-dataset.3.query.aql | 14 +
.../q01_pricing_summary_report_nt.1.ddl.aql | 33 ++
.../q01_pricing_summary_report_nt.2.update.aql | 12 +
.../q01_pricing_summary_report_nt.3.query.aql | 28 ++
.../scan-delete-rtree-secondary-index.1.ddl.aql | 26 ++
...an-delete-rtree-secondary-index.2.update.aql | 12 +
.../scan-delete-rtree-secondary-index.3.ddl.aql | 10 +
...an-delete-rtree-secondary-index.4.update.aql | 10 +
...can-delete-rtree-secondary-index.5.query.aql | 12 +
.../scan-insert-persistent-to-temp.1.ddl.aql | 35 +++
.../scan-insert-persistent-to-temp.2.update.aql | 17 ++
.../scan-insert-persistent-to-temp.3.ddl.aql | 12 +
.../scan-insert-persistent-to-temp.4.update.aql | 18 ++
.../scan-insert-persistent-to-temp.5.query.aql | 13 +
.../scan-insert-rtree-secondary-index.1.ddl.aql | 35 +++
...an-insert-rtree-secondary-index.2.update.aql | 17 ++
.../scan-insert-rtree-secondary-index.3.ddl.aql | 12 +
...an-insert-rtree-secondary-index.4.update.aql | 18 ++
...can-insert-rtree-secondary-index.5.query.aql | 13 +
.../scan-insert-temp-to-persistent.1.ddl.aql | 35 +++
.../scan-insert-temp-to-persistent.2.update.aql | 17 ++
.../scan-insert-temp-to-persistent.3.ddl.aql | 12 +
.../scan-insert-temp-to-persistent.4.update.aql | 18 ++
.../scan-insert-temp-to-persistent.5.query.aql | 13 +
.../compact-dataset-and-its-indexes.1.adm | 168 +++++++++++
.../delete-from-loaded-dataset-with-index.1.adm | 168 +++++++++++
.../delete-from-loaded-dataset.1.adm | 26 ++
.../drop-empty-secondary-indexes.1.adm | 1 +
.../temp-dataset/drop-index/drop-index.1.adm | 2 +
.../empty-load-with-index.1.adm | 2 +
.../insert-and-scan-dataset-with-index.1.adm | 241 +++++++++++++++
.../insert-and-scan-dataset.1.adm | 21 ++
.../q01_pricing_summary_report_nt.1.adm | 5 +
.../scan-delete-rtree-secondary-index.1.adm | 2 +
.../scan-insert-persistent-to-temp.1.adm | 4 +
.../scan-insert-rtree-secondary-index.1.adm | 4 +
.../scan-insert-persistent-to-temp.1.adm | 4 +
.../src/test/resources/runtimets/testsuite.xml | 62 ++++
.../aql/expression/ExternalDetailsDecl.java | 5 +
.../asterix/aql/expression/FeedDetailsDecl.java | 2 +-
.../aql/expression/IDatasetDetailsDecl.java | 2 +
.../aql/expression/InternalDetailsDecl.java | 10 +-
asterix-aql/src/main/javacc/AQL.html | 300 ++++++++++---------
asterix-aql/src/main/javacc/AQL.jj | 10 +-
.../asterix/common/config/DatasetConfig.java | 12 +-
.../common/context/DatasetLifecycleManager.java | 3 +-
.../context/PrimaryIndexOperationTracker.java | 11 +-
.../uci/ics/asterix/test/aql/TestsUtils.java | 18 +-
.../dataflow/HDFSLookupAdapterFactory.java | 11 +-
.../primary_index_only.1.script.aql | 1 +
.../primary_index_only.2.ddl.aql | 41 +++
.../primary_index_only.3.update.aql | 11 +
.../primary_index_only.4.txneu.aql | 22 ++
.../primary_index_only.5.txnqbc.aql | 10 +
.../primary_index_only.6.script.aql | 1 +
.../primary_index_only.7.script.aql | 1 +
.../primary_index_only.8.txnqar.aql | 10 +
.../primary_index_only.9.script.aql | 1 +
...ry_plus_default_secondary_index.1.script.aql | 1 +
...imary_plus_default_secondary_index.2.ddl.aql | 53 ++++
...ry_plus_default_secondary_index.3.update.aql | 11 +
...ary_plus_default_secondary_index.4.txneu.aql | 25 ++
...ry_plus_default_secondary_index.5.txnqbc.aql | 10 +
...ry_plus_default_secondary_index.6.script.aql | 1 +
...ry_plus_default_secondary_index.7.script.aql | 1 +
...ry_plus_default_secondary_index.8.txnqar.aql | 10 +
...ry_plus_default_secondary_index.9.script.aql | 1 +
...ry_plus_keyword_secondary_index.1.script.aql | 1 +
...imary_plus_keyword_secondary_index.2.ddl.aql | 54 ++++
...ry_plus_keyword_secondary_index.3.update.aql | 12 +
...ary_plus_keyword_secondary_index.4.txneu.aql | 25 ++
...ry_plus_keyword_secondary_index.5.txnqbc.aql | 5 +
...ry_plus_keyword_secondary_index.6.script.aql | 1 +
...ry_plus_keyword_secondary_index.7.script.aql | 1 +
...ry_plus_keyword_secondary_index.8.txnqar.aql | 5 +
...ry_plus_keyword_secondary_index.9.script.aql | 1 +
...plus_multiple_secondary_indices.1.script.aql | 1 +
...ry_plus_multiple_secondary_indices.2.ddl.aql | 60 ++++
...plus_multiple_secondary_indices.3.update.aql | 12 +
..._plus_multiple_secondary_indices.4.txneu.aql | 25 ++
...plus_multiple_secondary_indices.5.txnqbc.aql | 12 +
...plus_multiple_secondary_indices.6.script.aql | 1 +
...plus_multiple_secondary_indices.7.script.aql | 1 +
...plus_multiple_secondary_indices.8.txnqar.aql | 5 +
...plus_multiple_secondary_indices.9.script.aql | 1 +
.../primary_plus_ngram_index.1.script.aql | 1 +
.../primary_plus_ngram_index.2.ddl.aql | 53 ++++
.../primary_plus_ngram_index.3.update.aql | 11 +
.../primary_plus_ngram_index.4.txneu.aql | 25 ++
.../primary_plus_ngram_index.5.txnqbc.aql | 11 +
.../primary_plus_ngram_index.6.script.aql | 1 +
.../primary_plus_ngram_index.7.script.aql | 1 +
.../primary_plus_ngram_index.8.txnqar.aql | 11 +
.../primary_plus_ngram_index.9.script.aql | 1 +
.../primary_plus_rtree_index.1.script.aql | 1 +
.../primary_plus_rtree_index.2.ddl.aql | 54 ++++
.../primary_plus_rtree_index.3.update.aql | 12 +
.../primary_plus_rtree_index.4.txneu.aql | 25 ++
.../primary_plus_rtree_index.5.txnqbc.aql | 5 +
.../primary_plus_rtree_index.6.script.aql | 1 +
.../primary_plus_rtree_index.7.script.aql | 1 +
.../primary_plus_rtree_index.8.txnqar.aql | 5 +
.../primary_plus_rtree_index.9.script.aql | 1 +
.../dataset_recovery.1.script.aql | 1 +
.../dataset_recovery.2.ddl.aql | 19 ++
.../dataset_recovery.3.script.aql | 1 +
.../dataset_recovery.4.script.aql | 1 +
.../dataset_recovery.5.errddl.aql | 2 +
.../dataset_recovery.6.script.aql | 1 +
.../delete_after_recovery.1.script.aql | 1 +
.../delete_after_recovery.2.ddl.aql | 41 +++
.../delete_after_recovery.3.update.aql | 11 +
.../delete_after_recovery.4.script.aql | 1 +
.../delete_after_recovery.5.script.aql | 1 +
.../delete_after_recovery.6.update.aql | 5 +
.../delete_after_recovery.7.query.aql | 11 +
.../delete_after_recovery.8.script.aql | 1 +
.../insert_after_recovery.1.script.aql | 1 +
.../insert_after_recovery.2.ddl.aql | 41 +++
.../insert_after_recovery.3.update.aql | 11 +
.../insert_after_recovery.4.script.aql | 1 +
.../insert_after_recovery.5.script.aql | 1 +
.../insert_after_recovery.6.update.aql | 15 +
.../insert_after_recovery.7.query.aql | 11 +
.../insert_after_recovery.8.script.aql | 1 +
.../load_after_recovery.1.script.aql | 1 +
.../load_after_recovery.2.ddl.aql | 41 +++
.../load_after_recovery.3.script.aql | 1 +
.../load_after_recovery.4.script.aql | 1 +
.../load_after_recovery.5.update.aql | 11 +
.../load_after_recovery.6.query.aql | 10 +
.../load_after_recovery.7.script.aql | 1 +
.../secondary_index_recovery.1.script.aql | 1 +
.../secondary_index_recovery.2.ddl.aql | 21 ++
.../secondary_index_recovery.3.script.aql | 1 +
.../secondary_index_recovery.4.script.aql | 1 +
.../secondary_index_recovery.5.errddl.aql | 2 +
.../secondary_index_recovery.6.script.aql | 1 +
.../delete_after_recovery.1.adm | 2 +
.../insert_after_recovery.1.adm | 2 +
.../load_after_recovery.1.adm | 2 +
.../temp_primary_index_only/create_and_start.sh | 1 +
.../temp_primary_index_only/kill_cc_and_nc.sh | 1 +
.../temp_primary_index_only/stop_and_delete.sh | 3 +
.../temp_primary_index_only/stop_and_start.sh | 3 +
.../create_and_start.sh | 1 +
.../kill_cc_and_nc.sh | 1 +
.../stop_and_delete.sh | 3 +
.../stop_and_start.sh | 3 +
.../create_and_start.sh | 1 +
.../kill_cc_and_nc.sh | 1 +
.../stop_and_delete.sh | 3 +
.../stop_and_start.sh | 3 +
.../create_and_start.sh | 1 +
.../kill_cc_and_nc.sh | 1 +
.../stop_and_delete.sh | 3 +
.../stop_and_start.sh | 3 +
.../create_and_start.sh | 1 +
.../kill_cc_and_nc.sh | 1 +
.../stop_and_delete.sh | 3 +
.../stop_and_start.sh | 3 +
.../create_and_start.sh | 1 +
.../kill_cc_and_nc.sh | 1 +
.../stop_and_delete.sh | 3 +
.../stop_and_start.sh | 3 +
.../temp_dataset_recovery/create_and_start.sh | 1 +
.../temp_dataset_recovery/kill_cc_and_nc.sh | 1 +
.../temp_dataset_recovery/stop_and_delete.sh | 3 +
.../temp_dataset_recovery/stop_and_start.sh | 3 +
.../create_and_start.sh | 1 +
.../kill_cc_and_nc.sh | 1 +
.../stop_and_delete.sh | 3 +
.../stop_and_start.sh | 3 +
.../create_and_start.sh | 1 +
.../kill_cc_and_nc.sh | 1 +
.../stop_and_delete.sh | 3 +
.../stop_and_start.sh | 3 +
.../create_and_start.sh | 1 +
.../temp_load_after_recovery/kill_cc_and_nc.sh | 1 +
.../temp_load_after_recovery/stop_and_delete.sh | 3 +
.../temp_load_after_recovery/stop_and_start.sh | 3 +
.../create_and_start.sh | 1 +
.../kill_cc_and_nc.sh | 1 +
.../stop_and_delete.sh | 3 +
.../stop_and_start.sh | 3 +
.../test/resources/transactionts/testsuite.xml | 95 +++++-
.../ics/asterix/metadata/GarbageCollector.java | 39 +++
.../ics/asterix/metadata/IDatasetDetails.java | 25 ++
.../uci/ics/asterix/metadata/MetadataCache.java | 103 +++++--
.../ics/asterix/metadata/MetadataManager.java | 111 +++++--
.../metadata/MetadataTransactionContext.java | 3 +-
.../asterix/metadata/api/IMetadataManager.java | 24 +-
.../metadata/bootstrap/MetadataBootstrap.java | 8 +-
.../metadata/declared/AqlMetadataProvider.java | 136 +++++----
.../entities/ExternalDatasetDetails.java | 12 +
.../entities/InternalDatasetDetails.java | 17 +-
.../DatasetTupleTranslator.java | 11 +-
.../ExternalLoopkupOperatorDiscriptor.java | 9 +-
...tasetIndexModificationOperationCallback.java | 51 ++++
...dexModificationOperationCallbackFactory.java | 67 +++++
...dexModificationOperationCallbackFactory.java | 67 +++++
.../ExternalBTreeLocalResourceMetadata.java | 19 +-
...rnalBTreeWithBuddyLocalResourceMetadata.java | 21 +-
.../ExternalRTreeLocalResourceMetadata.java | 23 +-
.../resource/LSMBTreeLocalResourceMetadata.java | 24 +-
.../LSMInvertedIndexLocalResourceMetadata.java | 12 +-
.../resource/LSMRTreeLocalResourceMetadata.java | 24 +-
.../management/service/locking/LockManager.java | 31 +-
.../management/service/logging/LogPage.java | 4 +-
.../service/transaction/TransactionContext.java | 35 ++-
253 files changed, 3832 insertions(+), 478 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java
index 18c0793..ee71851 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitPOperator.java
@@ -18,6 +18,7 @@ package edu.uci.ics.asterix.algebra.operators.physical;
import java.util.List;
import edu.uci.ics.asterix.common.transactions.JobId;
+import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -29,7 +30,6 @@ import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSc
import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractPhysicalOperator;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
@@ -39,14 +39,11 @@ public class CommitPOperator extends AbstractPhysicalOperator {
private final List<LogicalVariable> primaryKeyLogicalVars;
private final JobId jobId;
private final int datasetId;
- private final boolean isWriteTransaction;
- public CommitPOperator(JobId jobId, int datasetId, List<LogicalVariable> primaryKeyLogicalVars,
- boolean isWriteTransaction) {
+ public CommitPOperator(JobId jobId, int datasetId, List<LogicalVariable> primaryKeyLogicalVars) {
this.jobId = jobId;
this.datasetId = datasetId;
this.primaryKeyLogicalVars = primaryKeyLogicalVars;
- this.isWriteTransaction = isWriteTransaction;
}
@Override
@@ -69,7 +66,7 @@ public class CommitPOperator extends AbstractPhysicalOperator {
public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
throws AlgebricksException {
AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
- deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+ deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
}
@Override
@@ -80,7 +77,10 @@ public class CommitPOperator extends AbstractPhysicalOperator {
RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema,
context);
int[] primaryKeyFields = JobGenHelper.variablesToFieldIndexes(primaryKeyLogicalVars, inputSchemas[0]);
- CommitRuntimeFactory runtime = new CommitRuntimeFactory(jobId, datasetId, primaryKeyFields, isWriteTransaction);
+
+ AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
+ CommitRuntimeFactory runtime = new CommitRuntimeFactory(jobId, datasetId, primaryKeyFields,
+ metadataProvider.isTemporaryDatasetWriteJob(), metadataProvider.isWriteTransaction());
builder.contributeMicroOperator(op, runtime, recDesc);
ILogicalOperator src = op.getInputs().get(0).getValue();
builder.contributeGraphEdge(src, 0, op, 0);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java
index af83b83..bed18fe 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntime.java
@@ -44,17 +44,17 @@ public class CommitRuntime implements IPushRuntime {
private final JobId jobId;
private final int datasetId;
private final int[] primaryKeyFields;
+ private final boolean isTemporaryDatasetWriteJob;
private final boolean isWriteTransaction;
private final long[] longHashes;
private final LogRecord logRecord;
private ITransactionContext transactionContext;
- private RecordDescriptor inputRecordDesc;
private FrameTupleAccessor frameTupleAccessor;
- private FrameTupleReference frameTupleReference;
+ private final FrameTupleReference frameTupleReference;
public CommitRuntime(IHyracksTaskContext ctx, JobId jobId, int datasetId, int[] primaryKeyFields,
- boolean isWriteTransaction) {
+ boolean isTemporaryDatasetWriteJob, boolean isWriteTransaction) {
this.hyracksTaskCtx = ctx;
IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) ctx.getJobletContext()
.getApplicationContext().getApplicationObject();
@@ -64,6 +64,7 @@ public class CommitRuntime implements IPushRuntime {
this.datasetId = datasetId;
this.primaryKeyFields = primaryKeyFields;
this.frameTupleReference = new FrameTupleReference();
+ this.isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob;
this.isWriteTransaction = isWriteTransaction;
this.longHashes = new long[2];
this.logRecord = new LogRecord();
@@ -85,14 +86,28 @@ public class CommitRuntime implements IPushRuntime {
frameTupleAccessor.reset(buffer);
int nTuple = frameTupleAccessor.getTupleCount();
for (int t = 0; t < nTuple; t++) {
- frameTupleReference.reset(frameTupleAccessor, t);
- pkHash = computePrimaryKeyHashValue(frameTupleReference, primaryKeyFields);
- logRecord.formEntityCommitLogRecord(transactionContext, datasetId, pkHash, frameTupleReference,
- primaryKeyFields);
- try {
- logMgr.log(logRecord);
- } catch (ACIDException e) {
- throw new HyracksDataException(e);
+ if (isTemporaryDatasetWriteJob) {
+ /**
+ * This "if branch" is for writes over temporary datasets.
+ * A temporary dataset does not require any lock and does not generate any write-ahead
+ * update and commit log but generates flush log and job commit log.
+ * However, a temporary dataset still MUST guarantee no-steal policy so that this
+ * notification call should be delivered to PrimaryIndexOptracker and used correctly in order
+ * to decrement number of active operation count of PrimaryIndexOptracker.
+ * By maintaining the count correctly and only allowing flushing when the count is 0, it can
+ * guarantee the no-steal policy for temporary datasets, too.
+ */
+ transactionContext.notifyOptracker(false);
+ } else {
+ frameTupleReference.reset(frameTupleAccessor, t);
+ pkHash = computePrimaryKeyHashValue(frameTupleReference, primaryKeyFields);
+ logRecord.formEntityCommitLogRecord(transactionContext, datasetId, pkHash, frameTupleReference,
+ primaryKeyFields);
+ try {
+ logMgr.log(logRecord);
+ } catch (ACIDException e) {
+ throw new HyracksDataException(e);
+ }
}
}
}
@@ -104,13 +119,12 @@ public class CommitRuntime implements IPushRuntime {
@Override
public void fail() throws HyracksDataException {
- // TODO Auto-generated method stub
}
@Override
public void close() throws HyracksDataException {
- // TODO Auto-generated method stub
+
}
@Override
@@ -120,7 +134,6 @@ public class CommitRuntime implements IPushRuntime {
@Override
public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor) {
- this.inputRecordDesc = recordDescriptor;
this.frameTupleAccessor = new FrameTupleAccessor(hyracksTaskCtx.getFrameSize(), recordDescriptor);
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java
index 6776f28..b996630 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/CommitRuntimeFactory.java
@@ -28,12 +28,15 @@ public class CommitRuntimeFactory implements IPushRuntimeFactory {
private final JobId jobId;
private final int datasetId;
private final int[] primaryKeyFields;
+ private final boolean isTemporaryDatasetWriteJob;
private final boolean isWriteTransaction;
- public CommitRuntimeFactory(JobId jobId, int datasetId, int[] primaryKeyFields, boolean isWriteTransaction) {
+ public CommitRuntimeFactory(JobId jobId, int datasetId, int[] primaryKeyFields, boolean isTemporaryDatasetWriteJob,
+ boolean isWriteTransaction) {
this.jobId = jobId;
this.datasetId = datasetId;
this.primaryKeyFields = primaryKeyFields;
+ this.isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob;
this.isWriteTransaction = isWriteTransaction;
}
@@ -44,6 +47,7 @@ public class CommitRuntimeFactory implements IPushRuntimeFactory {
@Override
public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) throws AlgebricksException {
- return new CommitRuntime(ctx, jobId, datasetId, primaryKeyFields, isWriteTransaction);
+ return new CommitRuntime(ctx, jobId, datasetId, primaryKeyFields, isTemporaryDatasetWriteJob,
+ isWriteTransaction);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
index ff21aff..67be923 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
@@ -172,10 +172,12 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
throw new AlgebricksException("Only record types can be indexed.");
}
ARecordType recordType = (ARecordType) itemType;
- Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypeEntries.get(0), secondaryKeyFieldEntries.get(0), recordType);
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypeEntries.get(0),
+ secondaryKeyFieldEntries.get(0), recordType);
IAType secondaryKeyType = keyPairType.first;
if (secondaryKeyType == null) {
- throw new AlgebricksException("Could not find field " + secondaryKeyFieldEntries.get(0) + " in the schema.");
+ throw new AlgebricksException("Could not find field " + secondaryKeyFieldEntries.get(0)
+ + " in the schema.");
}
// TODO: For now we assume the type of the generated tokens is the
@@ -236,8 +238,8 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(),
- datasetName, indexName);
+ .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), datasetName, indexName,
+ dataset.getDatasetDetails().isTemp());
// TODO: Here we assume there is only one search key field.
int queryField = keyFields[0];
// Get tokenizer and search modifier factories.
@@ -250,6 +252,7 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(
dataset, metadataProvider.getMetadataTxnContext());
+ boolean temp = dataset.getDatasetDetails().isTemp();
if (!isPartitioned) {
dataflowHelperFactory = new LSMInvertedIndexDataflowHelperFactory(
new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), compactionInfo.first,
@@ -258,7 +261,7 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps);
+ invertedIndexFieldsForNonBulkLoadOps, !temp);
} else {
dataflowHelperFactory = new PartitionedLSMInvertedIndexDataflowHelperFactory(
new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), compactionInfo.first,
@@ -267,7 +270,7 @@ public class InvertedIndexPOperator extends IndexSearchPOperator {
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps);
+ invertedIndexFieldsForNonBulkLoadOps, !temp);
}
LSMInvertedIndexSearchOperatorDescriptor invIndexSearchOp = new LSMInvertedIndexSearchOperatorDescriptor(
jobSpec, queryField, appContext.getStorageManagerInterface(), secondarySplitsAndConstraint.first,
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
index ef9d6cd..8de761a 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
@@ -99,8 +99,7 @@ public class ReplaceSinkOpWithCommitOpRule implements IAlgebraicRewriteRule {
//create the logical and physical operator
CommitOperator commitOperator = new CommitOperator(primaryKeyLogicalVars);
- CommitPOperator commitPOperator = new CommitPOperator(jobId, datasetId, primaryKeyLogicalVars,
- mp.isWriteTransaction());
+ CommitPOperator commitPOperator = new CommitPOperator(jobId, datasetId, primaryKeyLogicalVars);
commitOperator.setPhysicalOperator(commitPOperator);
//create ExtensionOperator and put the commitOperator in it.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
index 51fef41..0ce15ed 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
@@ -72,7 +72,6 @@ import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IOptimizationContextFactory;
import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
-import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
@@ -156,8 +155,7 @@ public class APIFramework {
}
public static Pair<Query, Integer> reWriteQuery(List<FunctionDecl> declaredFunctions,
- AqlMetadataProvider metadataProvider, Query q, SessionConfig conf)
- throws AsterixException {
+ AqlMetadataProvider metadataProvider, Query q, SessionConfig conf) throws AsterixException {
if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_EXPR_TREE)) {
conf.out().println();
@@ -185,8 +183,8 @@ public class APIFramework {
public static JobSpecification compileQuery(List<FunctionDecl> declaredFunctions,
AqlMetadataProvider queryMetadataProvider, Query rwQ, int varCounter, String outputDatasetName,
- SessionConfig conf, ICompiledDmlStatement statement)
- throws AsterixException, AlgebricksException, JSONException, RemoteException, ACIDException {
+ SessionConfig conf, ICompiledDmlStatement statement) throws AsterixException, AlgebricksException,
+ JSONException, RemoteException, ACIDException {
if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) {
conf.out().println();
@@ -219,7 +217,6 @@ public class APIFramework {
} else {
plan = t.translateLoad();
}
- boolean isWriteTransaction = queryMetadataProvider.isWriteTransaction();
LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
if (conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_LOGICAL_PLAN)) {
@@ -340,8 +337,8 @@ public class APIFramework {
builder.setTypeTraitProvider(format.getTypeTraitProvider());
builder.setNormalizedKeyComputerFactoryProvider(format.getNormalizedKeyComputerFactoryProvider());
- IJobletEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(asterixJobId,
- isWriteTransaction);
+ JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(asterixJobId,
+ queryMetadataProvider.isWriteTransaction());
JobSpecification spec = compiler.createJob(AsterixAppContextInfo.getInstance(), jobEventListenerFactory);
if (conf.is(SessionConfig.OOB_HYRACKS_JOB)) {
@@ -365,7 +362,7 @@ public class APIFramework {
}
public static void executeJobArray(IHyracksClientConnection hcc, JobSpecification[] specs, PrintWriter out)
- throws Exception {
+ throws Exception {
for (int i = 0; i < specs.length; i++) {
specs[i].setMaxReattempts(0);
JobId jobId = hcc.startJob(specs[i]);
@@ -378,8 +375,7 @@ public class APIFramework {
}
- public static void executeJobArray(IHyracksClientConnection hcc, Job[] jobs, PrintWriter out)
- throws Exception {
+ public static void executeJobArray(IHyracksClientConnection hcc, Job[] jobs, PrintWriter out) throws Exception {
for (int i = 0; i < jobs.length; i++) {
jobs[i].getJobSpec().setMaxReattempts(0);
long startTime = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
index 4c47c7d..e5ddf2b 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
@@ -159,7 +159,6 @@ import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.dataset.IHyracksDataset;
import edu.uci.ics.hyracks.api.dataset.ResultSetId;
import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
@@ -210,7 +209,7 @@ public class AqlTranslator extends AbstractAqlTranslator {
/**
* Compiles and submits for execution a list of AQL statements.
- *
+ *
* @param hcc
* A Hyracks client connection that is used to submit a jobspec to Hyracks.
* @param hdc
@@ -478,6 +477,7 @@ public class AqlTranslator extends AbstractAqlTranslator {
String compactionPolicy = dd.getDatasetDetailsDecl().getCompactionPolicy();
Map<String, String> compactionPolicyProperties = dd.getDatasetDetailsDecl().getCompactionPolicyProperties();
boolean defaultCompactionPolicy = (compactionPolicy == null);
+ boolean temp = dd.getDatasetDetailsDecl().isTemp();
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
@@ -539,7 +539,7 @@ public class AqlTranslator extends AbstractAqlTranslator {
datasetDetails = new InternalDatasetDetails(InternalDatasetDetails.FileStructure.BTREE,
InternalDatasetDetails.PartitioningStrategy.HASH, partitioningExprs, partitioningExprs,
partitioningTypes, ngName, autogenerated, compactionPolicy, compactionPolicyProperties,
- filterField);
+ filterField, temp);
break;
}
case EXTERNAL: {
@@ -2532,8 +2532,10 @@ public class AqlTranslator extends AbstractAqlTranslator {
throw new AlgebricksException("Tried to access non-existing dataset: " + datasetNameFrom);
}
+ Dataset datasetFrom = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseNameFrom, datasetNameFrom);
IFileSplitProvider fromSplits = metadataProvider.splitProviderAndPartitionConstraintsForDataset(
- dataverseNameFrom, datasetNameFrom, fromIndex.getIndexName()).first;
+ dataverseNameFrom, datasetNameFrom, fromIndex.getIndexName(), datasetFrom.getDatasetDetails()
+ .isTemp()).first;
StringBuilder fromSplitsPaths = new StringBuilder();
for (FileSplit f : fromSplits.getFileSplits()) {
@@ -2555,8 +2557,9 @@ public class AqlTranslator extends AbstractAqlTranslator {
throw new AlgebricksException("Tried to access non-existing dataset: " + datasetNameTo);
}
+ Dataset datasetTo = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseNameTo, datasetNameTo);
IFileSplitProvider toSplits = metadataProvider.splitProviderAndPartitionConstraintsForDataset(
- dataverseNameTo, datasetNameTo, toIndex.getIndexName()).first;
+ dataverseNameTo, datasetNameTo, toIndex.getIndexName(), datasetTo.getDatasetDetails().isTemp()).first;
StringBuilder toSplitsPaths = new StringBuilder();
for (FileSplit f : toSplits.getFileSplits()) {
@@ -2573,7 +2576,8 @@ public class AqlTranslator extends AbstractAqlTranslator {
IDatasetDetailsDecl idd = new InternalDetailsDecl(new Identifier(toDataset.getDatasetDetails()
.getNodeGroupName()), toIndex.getKeyFieldNames(), false, toDataset.getDatasetDetails()
- .getCompactionPolicy(), toDataset.getDatasetDetails().getCompactionPolicyProperties(), null);
+ .getCompactionPolicy(), toDataset.getDatasetDetails().getCompactionPolicyProperties(), null,
+ toDataset.getDatasetDetails().isTemp());
DatasetDecl createToDataset = new DatasetDecl(new Identifier(dataverseNameTo),
pregelixStmt.getDatasetNameTo(), new Identifier(toDataset.getItemTypeName()),
toDataset.getHints(), toDataset.getDatasetType(), idd, false);
@@ -2691,19 +2695,21 @@ public class AqlTranslator extends AbstractAqlTranslator {
new IPushRuntimeFactory[] { new EmptyTupleSourceRuntimeFactory() }, rDescs);
edu.uci.ics.asterix.common.transactions.JobId jobId = JobIdFactory.generateJobId();
+ Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
FlushDatasetOperatorDescriptor flushOperator = new FlushDatasetOperatorDescriptor(spec, jobId,
- MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName).getDatasetId());
+ dataset.getDatasetId());
spec.connect(new OneToOneConnectorDescriptor(spec), emptySource, 0, flushOperator, 0);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName);
+ .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, dataset
+ .getDatasetDetails().isTemp());
AlgebricksPartitionConstraint primaryPartitionConstraint = primarySplitsAndConstraint.second;
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, emptySource,
primaryPartitionConstraint);
- IJobletEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(jobId, false);
+ JobEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(jobId, true);
spec.setJobletEventListenerFactory(jobEventListenerFactory);
runJob(hcc, spec, true);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
index 7a9e1b2..b2da545 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
@@ -86,6 +86,7 @@ public class DatasetOperations {
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
return JobSpecificationUtils.createJobSpecification();
}
+ boolean temp = dataset.getDatasetDetails().isTemp();
Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(),
dataverseName);
@@ -106,11 +107,13 @@ public class DatasetOperations {
JobSpecification specPrimary = JobSpecificationUtils.createJobSpecification();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), datasetName,
- datasetName);
+ .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), datasetName, datasetName,
+ temp);
AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
metadataProvider.getMetadataTxnContext());
+
+ // The index drop operation should be persistent regardless of temp datasets or permanent dataset
IndexDropOperatorDescriptor primaryBtreeDrop = new IndexDropOperatorDescriptor(specPrimary,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
@@ -118,7 +121,7 @@ public class DatasetOperations {
new PrimaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), true, filterTypeTraits,
- filterCmpFactories, btreeFields, filterFields));
+ filterCmpFactories, btreeFields, filterFields, true));
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(specPrimary, primaryBtreeDrop,
splitsAndConstraint.second);
@@ -140,6 +143,7 @@ public class DatasetOperations {
if (dataset == null) {
throw new AsterixException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
}
+ boolean temp = dataset.getDatasetDetails().isTemp();
ARecordType itemType = (ARecordType) metadata.findType(dataverseName, dataset.getItemTypeName());
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
@@ -154,7 +158,7 @@ public class DatasetOperations {
int[] btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadata
- .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
+ .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName, temp);
FileSplit[] fs = splitsAndConstraint.first.getFileSplits();
StringBuilder sb = new StringBuilder();
for (int i = 0; i < fs.length; i++) {
@@ -172,6 +176,7 @@ public class DatasetOperations {
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
+ // The index create operation should be persistent regardless of temp datasets or permanent dataset
TreeIndexCreateOperatorDescriptor indexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
splitsAndConstraint.first, typeTraits, comparatorFactories, bloomFilterKeyFields,
@@ -180,7 +185,7 @@ public class DatasetOperations {
.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
.getBloomFilterFalsePositiveRate(), true, filterTypeTraits, filterCmpFactories,
- btreeFields, filterFields), localResourceFactoryProvider,
+ btreeFields, filterFields, true), localResourceFactoryProvider,
NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, indexCreateOp,
splitsAndConstraint.second);
@@ -205,6 +210,8 @@ public class DatasetOperations {
if (dataset == null) {
throw new AsterixException("Could not find dataset " + datasetName + " in dataverse " + dataverseName);
}
+ boolean temp = dataset.getDatasetDetails().isTemp();
+
ARecordType itemType = (ARecordType) metadata.findType(dataverseName, dataset.getItemTypeName());
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
@@ -219,7 +226,7 @@ public class DatasetOperations {
int[] btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset);
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadata
- .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
+ .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName, temp);
AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
@@ -233,7 +240,7 @@ public class DatasetOperations {
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), true, filterTypeTraits,
- filterCmpFactories, btreeFields, filterFields), NoOpOperationCallbackFactory.INSTANCE);
+ filterCmpFactories, btreeFields, filterFields, !temp), NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper
.setPartitionConstraintInJobSpec(spec, compactOp, splitsAndConstraint.second);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
index ba6357b..a519eaf 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
@@ -212,7 +212,7 @@ public class ExternalIndexingOperations {
public static FileSystem getFileSystemObject(Map<String, String> map) throws IOException {
Configuration conf = new Configuration();
- conf.set("fs.default.name", ((String) map.get(HDFSAdapterFactory.KEY_HDFS_URL)).trim());
+ conf.set("fs.default.name", map.get(HDFSAdapterFactory.KEY_HDFS_URL).trim());
conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
return FileSystem.get(conf);
}
@@ -241,7 +241,7 @@ public class ExternalIndexingOperations {
mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
ExternalFilesIndexOperatorDescriptor externalFilesOp = new ExternalFilesIndexOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
secondaryFileSplitProvider, indexDataflowHelperFactory, localResourceFactoryProvider,
@@ -408,6 +408,7 @@ public class ExternalIndexingOperations {
: indexDropStmt.getDataverseName();
String datasetName = indexDropStmt.getDatasetName();
String indexName = indexDropStmt.getIndexName();
+ boolean temp = dataset.getDatasetDetails().isTemp();
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForFilesIndex(dataverseName, datasetName, indexName, true);
@@ -420,7 +421,7 @@ public class ExternalIndexingOperations {
dataset.getDatasetId()), compactionInfo.first, compactionInfo.second,
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
- storageProperties.getBloomFilterFalsePositiveRate(), false, null, null, null, null));
+ storageProperties.getBloomFilterFalsePositiveRate(), false, null, null, null, null, !temp));
AlgebricksPartitionConstraintHelper
.setPartitionConstraintInJobSpec(spec, btreeDrop, splitsAndConstraint.second);
spec.addRoot(btreeDrop);
@@ -488,11 +489,12 @@ public class ExternalIndexingOperations {
AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(ds,
metadataProvider.getMetadataTxnContext());
+ boolean temp = ds.getDatasetDetails().isTemp();
ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
- getFilesIndexName(ds.getDatasetName()));
+ getFilesIndexName(ds.getDatasetName()), temp);
IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
@@ -508,7 +510,7 @@ public class ExternalIndexingOperations {
if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
- index.getIndexName());
+ index.getIndexName(), temp);
if (index.getIndexType() == IndexType.BTREE) {
btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
mergePolicyFactoryProperties, storageProperties, spec));
@@ -543,7 +545,7 @@ public class ExternalIndexingOperations {
new SecondaryIndexOperationTrackerProvider(ds.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(),
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds), true);
}
private static ExternalBTreeWithBuddyDataflowHelperFactory getBTreeDataflowHelperFactory(Dataset ds, Index index,
@@ -554,7 +556,7 @@ public class ExternalIndexingOperations {
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), new int[] { index.getKeyFieldNames().size() },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds), true);
}
@SuppressWarnings("rawtypes")
@@ -609,7 +611,7 @@ public class ExternalIndexingOperations {
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), new int[] { index.getKeyFieldNames().size() },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds), true);
}
public static JobSpecification buildAbortOp(Dataset ds, List<Index> indexes, AqlMetadataProvider metadataProvider)
@@ -621,9 +623,11 @@ public class ExternalIndexingOperations {
metadataProvider.getMetadataTxnContext());
ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+
+ boolean temp = ds.getDatasetDetails().isTemp();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
- getFilesIndexName(ds.getDatasetName()));
+ getFilesIndexName(ds.getDatasetName()), temp);
IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
@@ -639,7 +643,7 @@ public class ExternalIndexingOperations {
if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
- index.getIndexName());
+ index.getIndexName(), temp);
if (index.getIndexType() == IndexType.BTREE) {
btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
mergePolicyFactoryProperties, storageProperties, spec));
@@ -677,9 +681,11 @@ public class ExternalIndexingOperations {
metadataProvider.getMetadataTxnContext());
ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+ boolean temp = ds.getDatasetDetails().isTemp();
+
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
- getFilesIndexName(ds.getDatasetName()));
+ getFilesIndexName(ds.getDatasetName()), temp);
IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
@@ -695,7 +701,7 @@ public class ExternalIndexingOperations {
if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
- index.getIndexName());
+ index.getIndexName(), temp);
if (index.getIndexType() == IndexType.BTREE) {
btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
mergePolicyFactoryProperties, storageProperties, spec));
@@ -740,7 +746,7 @@ public class ExternalIndexingOperations {
mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
secondaryFileSplitProvider, FilesIndexDescription.EXTERNAL_FILE_INDEX_TYPE_TRAITS,
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
index 53dfc6e..5159277 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
@@ -94,19 +94,22 @@ public class IndexOperations {
String datasetName = indexDropStmt.getDatasetName();
String indexName = indexDropStmt.getIndexName();
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+ boolean temp = dataset.getDatasetDetails().isTemp();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName);
+ .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, temp);
AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
metadataProvider.getMetadataTxnContext());
+
+ // The index drop operation should be persistent regardless of temp datasets or permanent dataset.
IndexDropOperatorDescriptor btreeDrop = new IndexDropOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
dataset.getDatasetId()), compactionInfo.first, compactionInfo.second,
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
- storageProperties.getBloomFilterFalsePositiveRate(), false, null, null, null, null));
+ storageProperties.getBloomFilterFalsePositiveRate(), false, null, null, null, null, true));
AlgebricksPartitionConstraintHelper
.setPartitionConstraintInJobSpec(spec, btreeDrop, splitsAndConstraint.second);
spec.addRoot(btreeDrop);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
index a061451..333d6e7 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
@@ -92,12 +92,13 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
secondaryBTreeFields, secondaryFilterFields);
localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
LocalResource.LSMBTreeResource);
+ // The index create operation should be persistent regardless of temp datasets or permanent dataset.
indexDataflowHelperFactory = new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), false, filterTypeTraits, filterCmpFactories,
- secondaryBTreeFields, secondaryFilterFields);
+ secondaryBTreeFields, secondaryFilterFields, true);
} else {
// External dataset local resource and dataflow helper
int[] buddyBreeFields = new int[] { numSecondaryKeys };
@@ -111,7 +112,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), buddyBreeFields,
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
}
TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
@@ -163,7 +164,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numSecondaryKeys },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
IOperatorDescriptor root;
if (externalFiles != null) {
// Transaction load
@@ -216,6 +217,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+ boolean temp = dataset.getDatasetDetails().isTemp();
// Create secondary BTree bulk load op.
TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
spec,
@@ -226,7 +228,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
.getBloomFilterFalsePositiveRate(), false, filterTypeTraits, filterCmpFactories,
- secondaryBTreeFields, secondaryFilterFields), GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
+ secondaryBTreeFields, secondaryFilterFields, !temp), GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
new IPushRuntimeFactory[] { new SinkRuntimeFactory() }, new RecordDescriptor[] { secondaryRecDesc });
@@ -247,6 +249,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
}
}
+ @Override
protected int getNumSecondaryKeys() {
return numSecondaryKeys;
}
@@ -256,6 +259,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+ boolean temp = dataset.getDatasetDetails().isTemp();
LSMTreeIndexCompactOperatorDescriptor compactOp;
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
@@ -267,7 +271,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), false, filterTypeTraits,
- filterCmpFactories, secondaryBTreeFields, secondaryFilterFields),
+ filterCmpFactories, secondaryBTreeFields, secondaryFilterFields, !temp),
NoOpOperationCallbackFactory.INSTANCE);
} else {
// External dataset
@@ -280,7 +284,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE, storageProperties
.getBloomFilterFalsePositiveRate(), new int[] { numSecondaryKeys },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset)),
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true),
NoOpOperationCallbackFactory.INSTANCE);
}
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
index cccb461..07c8bab 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
@@ -148,8 +148,8 @@ public abstract class SecondaryIndexOperationsHelper {
}
public static SecondaryIndexOperationsHelper createIndexOperationsHelper(IndexType indexType, String dataverseName,
- String datasetName, String indexName, List<List<String>> secondaryKeyFields, List<IAType> secondaryKeyTypes,
- boolean isEnforced, int gramLength, AqlMetadataProvider metadataProvider,
+ String datasetName, String indexName, List<List<String>> secondaryKeyFields,
+ List<IAType> secondaryKeyTypes, boolean isEnforced, int gramLength, AqlMetadataProvider metadataProvider,
PhysicalOptimizationConfig physOptConf, ARecordType recType, ARecordType enforcedType)
throws AsterixException, AlgebricksException {
IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
@@ -198,12 +198,13 @@ public abstract class SecondaryIndexOperationsHelper {
if (dataset == null) {
throw new AsterixException("Unknown dataset " + datasetName);
}
+ boolean temp = dataset.getDatasetDetails().isTemp();
itemType = aRecType;
enforcedItemType = enforcedType;
payloadSerde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
numSecondaryKeys = secondaryKeyFields.size();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, secondaryIndexName);
+ .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, secondaryIndexName, temp);
secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
secondaryPartitionConstraint = secondarySplitsAndConstraint.second;
@@ -219,7 +220,7 @@ public abstract class SecondaryIndexOperationsHelper {
numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
+ .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName, temp);
primaryFileSplitProvider = primarySplitsAndConstraint.first;
primaryPartitionConstraint = primarySplitsAndConstraint.second;
setPrimaryRecDescAndComparators();
@@ -291,9 +292,9 @@ public abstract class SecondaryIndexOperationsHelper {
primaryRecDesc = new RecordDescriptor(primaryRecFields, primaryTypeTraits);
}
- protected abstract void setSecondaryRecDescAndComparators(IndexType indexType, List<List<String>> secondaryKeyFields,
- List<IAType> secondaryKeyTypes, int gramLength, AqlMetadataProvider metadataProvider)
- throws AlgebricksException, AsterixException;
+ protected abstract void setSecondaryRecDescAndComparators(IndexType indexType,
+ List<List<String>> secondaryKeyFields, List<IAType> secondaryKeyTypes, int gramLength,
+ AqlMetadataProvider metadataProvider) throws AlgebricksException, AsterixException;
protected AbstractOperatorDescriptor createDummyKeyProviderOp(JobSpecification spec) throws AsterixException,
AlgebricksException {
@@ -330,9 +331,10 @@ public abstract class SecondaryIndexOperationsHelper {
IJobletEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(jobId, isWriteTransaction);
spec.setJobletEventListenerFactory(jobEventListenerFactory);
- ISearchOperationCallbackFactory searchCallbackFactory = new PrimaryIndexInstantSearchOperationCallbackFactory(
- jobId, dataset.getDatasetId(), primaryBloomFilterKeyFields, txnSubsystemProvider,
- ResourceType.LSM_BTREE);
+ boolean temp = dataset.getDatasetDetails().isTemp();
+ ISearchOperationCallbackFactory searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+ : new PrimaryIndexInstantSearchOperationCallbackFactory(jobId, dataset.getDatasetId(),
+ primaryBloomFilterKeyFields, txnSubsystemProvider, ResourceType.LSM_BTREE);
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
@@ -343,7 +345,7 @@ public abstract class SecondaryIndexOperationsHelper {
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
.getBloomFilterFalsePositiveRate(), true, filterTypeTraits, filterCmpFactories,
- primaryBTreeFields, primaryFilterFields), false, false, null,
+ primaryBTreeFields, primaryFilterFields, !temp), false, false, null,
searchCallbackFactory, null, null);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, primarySearchOp,
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
index 579af2f..74c4256 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexOperationsHelper.java
@@ -203,6 +203,7 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryIndexOperat
}
+ @Override
protected int getNumSecondaryKeys() {
return numTokenKeyPairFields - numPrimaryKeys;
}
@@ -330,6 +331,7 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryIndexOperat
private IIndexDataflowHelperFactory createDataflowHelperFactory() {
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+ boolean temp = dataset.getDatasetDetails().isTemp();
if (!isPartitioned) {
return new LSMInvertedIndexDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
@@ -338,7 +340,7 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryIndexOperat
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, secondaryFilterFields, secondaryFilterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps);
+ invertedIndexFieldsForNonBulkLoadOps, !temp);
} else {
return new PartitionedLSMInvertedIndexDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
@@ -347,7 +349,7 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryIndexOperat
LSMInvertedIndexIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits,
filterCmpFactories, secondaryFilterFields, secondaryFilterFieldsForNonBulkLoadOps,
- invertedIndexFieldsForNonBulkLoadOps);
+ invertedIndexFieldsForNonBulkLoadOps, !temp);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/ccd67fe8/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
index 8be9f59..0e49fb9 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
@@ -92,7 +92,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-
+ boolean temp = dataset.getDatasetDetails().isTemp();
IIndexDataflowHelperFactory indexDataflowHelperFactory;
ILocalResourceFactoryProvider localResourceFactoryProvider;
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
@@ -112,7 +112,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, primaryKeyFields,
- filterTypeTraits, filterCmpFactories, secondaryFilterFields);
+ filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp);
} else {
// External dataset
// Prepare a LocalResourceMetadata which will be stored in NC's local resource repository
@@ -131,7 +131,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
secondaryComparatorFactories.length), storageProperties.getBloomFilterFalsePositiveRate(),
new int[] { numNestedSecondaryKeyFields },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
}
TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
@@ -146,6 +146,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
return spec;
}
+ @Override
protected int getNumSecondaryKeys() {
return numNestedSecondaryKeyFields;
}
@@ -233,6 +234,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
@Override
public JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+ boolean temp = dataset.getDatasetDetails().isTemp();
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
// Create dummy key provider for feeding the primary index scan.
AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);
@@ -273,7 +275,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
secondaryComparatorFactories.length), storageProperties
.getBloomFilterFalsePositiveRate(), rtreeFields, primaryKeyFields,
- filterTypeTraits, filterCmpFactories, secondaryFilterFields),
+ filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp),
GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
new IPushRuntimeFactory[] { new SinkRuntimeFactory() }, new RecordDescriptor[] {});
@@ -325,7 +327,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numNestedSecondaryKeyFields },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
// Create secondary RTree bulk load op.
IOperatorDescriptor root;
AbstractTreeIndexOperatorDescriptor secondaryBulkLoadOp;
@@ -364,6 +366,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+ boolean temp = dataset.getDatasetDetails().isTemp();
LSMTreeIndexCompactOperatorDescriptor compactOp;
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
@@ -377,7 +380,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
secondaryComparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, primaryKeyFields,
- filterTypeTraits, filterCmpFactories, secondaryFilterFields),
+ filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp),
NoOpOperationCallbackFactory.INSTANCE);
} else {
// External dataset
@@ -392,7 +395,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
secondaryComparatorFactories.length), storageProperties
.getBloomFilterFalsePositiveRate(), new int[] { numNestedSecondaryKeyFields },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset)),
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true),
NoOpOperationCallbackFactory.INSTANCE);
}