You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by "Wenhai Li (Code Review)" <do...@asterixdb.incubator.apache.org> on 2015/10/02 09:00:20 UTC

Change in asterixdb[master]: PLEASE EDIT to provide a meaningful commit message!

Wenhai Li has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/428

Change subject: PLEASE EDIT to provide a meaningful commit message!
......................................................................

PLEASE EDIT to provide a meaningful commit message!

The following commits from your working branch will be included:
exchange optimization for lookup and delete when the search condition is merely composed of the single primary key.

commit 68704b9dd1afc477929339d1b453fbb2f548804d
Author: Michael <lw...@yahoo.com>
Date:   Fri Oct 2 00:01:16 2015 -0700

    exchange optimization for lookup and delete

Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
---
M asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
M pom.xml
3 files changed, 406 insertions(+), 7 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/28/428/1

diff --git a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
index da7d9c6..bc3fd47 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
@@ -18,14 +18,25 @@
  */
 package org.apache.asterix.algebra.operators.physical;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.asterix.formats.base.IDataFormat;
+import org.apache.asterix.metadata.MetadataException;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataTransactionContext;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.declared.AqlSourceId;
 import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.utils.DatasetUtils;
 import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.optimizer.rules.am.BTreeJobGenParams;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.common.utils.ListSet;
@@ -34,12 +45,14 @@
 import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
 import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
 import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -54,7 +67,15 @@
 import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
 import org.apache.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
 import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import org.apache.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
 import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.api.dataflow.value.IBinaryHashFunction;
+import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
 
 /**
  * Contributes the runtime operator for an unnest-map representing a BTree search.
@@ -90,6 +111,81 @@
         return PhysicalOperatorTag.BTREE_SEARCH;
     }
 
+    private int extractPrimaryKeyNodeId(JobGenContext context, IOperatorSchema[] inputSchemas, ILogicalOperator op)
+            throws AlgebricksException, MetadataException, IOException, InstantiationException, IllegalAccessException,
+            ClassNotFoundException {
+        int route = -1;
+        boolean found = false;
+        ILogicalOperator pop = op;
+        List<LogicalVariable> lv = new ArrayList<LogicalVariable>();
+        List<Mutable<ILogicalExpression>> le = null;
+
+        while (pop.hasInputs()) {
+            pop = pop.getInputs().get(0).getValue();
+            if (LogicalOperatorTag.ASSIGN == pop.getOperatorTag()) {
+                VariableUtilities.getProducedVariables(pop, lv);
+                if (lowKeyVarList.size() != lv.size() / 2)
+                    continue;
+                boolean isPrimaryKey = true;
+                for (int i = 0; i < lowKeyVarList.size(); i++) {
+                    if (lowKeyVarList.get(i).getId() != lv.get(i).getId())
+                        isPrimaryKey = false;
+                }
+                if (isPrimaryKey) {
+                    found = true;
+                    VariableUtilities.getLiveVariables(pop, lv);
+                    le = ((AssignOperator) pop).getExpressions();
+                    break;
+                }
+            }
+        }
+
+        if (found) {
+            boolean isEqual = true;
+            for (int i = 0; i < le.size() / 2; i++) {
+                if (le.get(i).getValue() != le.get(i + le.size() / 2).getValue())
+                    isEqual = false;
+            }
+            if (isEqual && isEqCondition) {
+                int nPartition = ((AlgebricksAbsolutePartitionConstraint) (context.getClusterLocations()))
+                        .getLocations().length;
+                MetadataTransactionContext ctx = ((AqlMetadataProvider) (context.getMetadataProvider()))
+                        .getMetadataTxnContext();
+                Dataset dataset = MetadataManager.INSTANCE.getDataset(ctx, idx.getDataSource().getId()
+                        .getDataverseName(), idx.getDataSource().getId().getDatasourceName());
+                IAType type = MetadataManager.INSTANCE.getDatatype(ctx, idx.getDataSource().getId().getDataverseName(),
+                        dataset.getItemTypeName()).getDatatype();
+                IDataFormat format = (IDataFormat) Class.forName(
+                        MetadataManager.INSTANCE.getDataverse(ctx, idx.getDataSource().getId().getDataverseName())
+                                .getDataFormat()).newInstance();
+                IBinaryHashFunctionFactory[] hashFuns = DatasetUtils.computeKeysBinaryHashFunFactories(dataset,
+                        (ARecordType) type, format.getBinaryHashFunctionFactoryProvider());
+                int h = 0;
+                if (hashFuns.length == le.size() / 2) {
+                    for (int i = 0; i < hashFuns.length; i++) {
+                        IBinaryHashFunction bhf = hashFuns[i].createBinaryHashFunction();
+                        ILogicalExpression expr = le.get(i).getValue();
+                        IScalarEvaluatorFactory ief = context.getExpressionRuntimeProvider().createEvaluatorFactory(
+                                expr, context.getTypeEnvironment(op), inputSchemas, context);
+                        ByteBufferInputStream buffer = new ByteBufferInputStream();
+                        IScalarEvaluator eval = ief.createScalarEvaluator(null);
+                        IPointable p = VoidPointable.FACTORY.createPointable();
+                        eval.evaluate(null, p);
+                        buffer.setByteBuffer(ByteBuffer.wrap(p.getByteArray(), p.getStartOffset(), p.getLength()), 0);
+                        int fh = bhf.hash(p.getByteArray(), p.getStartOffset(), p.getLength());
+                        h = h * 31 + fh;
+                        buffer.close();
+                    }
+                    if (h < 0)
+                        h = -(h + 1);
+                    route = h % nPartition;
+                }
+            }
+        }
+
+        return route;
+    }
+
     @Override
     public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
             IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
@@ -120,11 +216,34 @@
             outputVars = new ArrayList<LogicalVariable>();
             VariableUtilities.getLiveVariables(unnestMap, outputVars);
         }
-        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> btreeSearch = metadataProvider.buildBtreeRuntime(
-                builder.getJobSpec(), outputVars, opSchema, typeEnv, context, jobGenParams.getRetainInput(),
-                jobGenParams.getRetainNull(), dataset, jobGenParams.getIndexName(), lowKeyIndexes, highKeyIndexes,
-                jobGenParams.isLowKeyInclusive(), jobGenParams.isHighKeyInclusive(), implConfig, minFilterFieldIndexes,
-                maxFilterFieldIndexes);
+        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> btreeSearch = null;
+        if (isPrimaryIndex) {
+            int nodeId = -1;
+            try {
+                nodeId = extractPrimaryKeyNodeId(context, inputSchemas, op);
+            } catch (MetadataException e) {
+                throw new AlgebricksException("Extract primary key metadata error: " + op.toString());
+            } catch (IOException e) {
+                throw new AlgebricksException("Extract primary key io error: " + op.toString());
+            } catch (InstantiationException e) {
+                throw new AlgebricksException("Extract primary key instantiation error: " + op.toString());
+            } catch (IllegalAccessException e) {
+                throw new AlgebricksException("Extract primary key access error: " + op.toString());
+            } catch (ClassNotFoundException e) {
+                throw new AlgebricksException("Extract primary key class finding error: " + op.toString());
+            }
+            btreeSearch = metadataProvider
+                    .buildBtreeRuntime(builder.getJobSpec(), outputVars, opSchema, typeEnv, context,
+                            jobGenParams.getRetainInput(), jobGenParams.getRetainNull(), dataset,
+                            jobGenParams.getIndexName(), lowKeyIndexes, highKeyIndexes,
+                            jobGenParams.isLowKeyInclusive(), jobGenParams.isHighKeyInclusive(), implConfig,
+                            minFilterFieldIndexes, maxFilterFieldIndexes, nodeId);
+        } else {
+            btreeSearch = metadataProvider.buildBtreeRuntime(builder.getJobSpec(), outputVars, opSchema, typeEnv,
+                    context, jobGenParams.getRetainInput(), jobGenParams.getRetainNull(), dataset,
+                    jobGenParams.getIndexName(), lowKeyIndexes, highKeyIndexes, jobGenParams.isLowKeyInclusive(),
+                    jobGenParams.isHighKeyInclusive(), implConfig, minFilterFieldIndexes, maxFilterFieldIndexes);
+        }
 
         builder.contributeHyracksOperator(unnestMap, btreeSearch.first);
         builder.contributeAlgebricksPartitionConstraint(btreeSearch.first, btreeSearch.second);
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
index 2478dc7..e1c46cd 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
@@ -60,6 +60,7 @@
 import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
 import org.apache.asterix.formats.nontagged.AqlLinearizeComparatorFactoryProvider;
 import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
+import org.apache.asterix.metadata.IDatasetDetails;
 import org.apache.asterix.metadata.MetadataException;
 import org.apache.asterix.metadata.MetadataManager;
 import org.apache.asterix.metadata.MetadataTransactionContext;
@@ -672,6 +673,152 @@
             List<LogicalVariable> outputVars, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv,
             JobGenContext context, boolean retainInput, boolean retainNull, Dataset dataset, String indexName,
             int[] lowKeyFields, int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive,
+            Object implConfig, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes, int nodeId)
+            throws AlgebricksException {
+
+        boolean isSecondary = true;
+        int numSecondaryKeys = 0;
+        try {
+            boolean temp = dataset.getDatasetDetails().isTemp();
+            Index primaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
+                    dataset.getDatasetName(), dataset.getDatasetName());
+            if (primaryIndex != null && dataset.getDatasetType() != DatasetType.EXTERNAL) {
+                isSecondary = !indexName.equals(primaryIndex.getIndexName());
+            }
+            int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
+            RecordDescriptor outputRecDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
+            int[] bloomFilterKeyFields;
+            ITypeTraits[] typeTraits;
+            IBinaryComparatorFactory[] comparatorFactories;
+
+            String itemTypeName = dataset.getItemTypeName();
+            ARecordType itemType = (ARecordType) MetadataManager.INSTANCE.getDatatype(mdTxnCtx,
+                    dataset.getDataverseName(), itemTypeName).getDatatype();
+            ITypeTraits[] filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, itemType);
+            IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(
+                    dataset, itemType, context.getBinaryComparatorFactoryProvider());
+            int[] filterFields = null;
+            int[] btreeFields = null;
+
+            if (isSecondary) {
+                Index secondaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
+                        dataset.getDatasetName(), indexName);
+                numSecondaryKeys = secondaryIndex.getKeyFieldNames().size();
+                bloomFilterKeyFields = new int[numSecondaryKeys];
+                for (int i = 0; i < numSecondaryKeys; i++) {
+                    bloomFilterKeyFields[i] = i;
+                }
+                typeTraits = JobGenHelper.variablesToTypeTraits(outputVars, 0, outputVars.size(), typeEnv, context);
+                comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(outputVars, 0,
+                        outputVars.size(), typeEnv, context);
+
+                if (filterTypeTraits != null) {
+                    filterFields = new int[1];
+                    filterFields[0] = numSecondaryKeys + numPrimaryKeys;
+                    btreeFields = new int[numSecondaryKeys + numPrimaryKeys];
+                    for (int k = 0; k < btreeFields.length; k++) {
+                        btreeFields[k] = k;
+                    }
+                }
+
+            } else {
+                bloomFilterKeyFields = new int[numPrimaryKeys];
+                for (int i = 0; i < numPrimaryKeys; i++) {
+                    bloomFilterKeyFields[i] = i;
+                }
+
+                typeTraits = DatasetUtils.computeTupleTypeTraits(dataset, itemType);
+                comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset, itemType,
+                        context.getBinaryComparatorFactoryProvider());
+
+                filterFields = DatasetUtils.createFilterFields(dataset);
+                btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset);
+            }
+
+            IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
+            Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spPc;
+            try {
+                Index curIdx = MetadataManager.INSTANCE.getIndex(getMetadataTxnContext(), dataset.getDataverseName(),
+                        dataset.getDatasetName(), indexName);
+                if (null != curIdx && curIdx.isPrimaryIndex() && nodeId != -1)
+                    spPc = splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(),
+                            dataset.getDatasetName(), indexName, temp, nodeId);
+                else
+                    spPc = splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(),
+                            dataset.getDatasetName(), indexName, temp);
+            } catch (Exception e) {
+                throw new AlgebricksException(e);
+            }
+
+            ISearchOperationCallbackFactory searchCallbackFactory = null;
+            if (isSecondary) {
+                searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+                        : new SecondaryIndexSearchOperationCallbackFactory();
+            } else {
+                JobId jobId = ((JobEventListenerFactory) jobSpec.getJobletEventListenerFactory()).getJobId();
+                int datasetId = dataset.getDatasetId();
+                int[] primaryKeyFields = new int[numPrimaryKeys];
+                for (int i = 0; i < numPrimaryKeys; i++) {
+                    primaryKeyFields[i] = i;
+                }
+
+                AqlMetadataImplConfig aqlMetadataImplConfig = (AqlMetadataImplConfig) implConfig;
+                ITransactionSubsystemProvider txnSubsystemProvider = new TransactionSubsystemProvider();
+                if (aqlMetadataImplConfig != null && aqlMetadataImplConfig.isInstantLock()) {
+                    searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+                            : new PrimaryIndexInstantSearchOperationCallbackFactory(jobId, datasetId, primaryKeyFields,
+                                    txnSubsystemProvider, ResourceType.LSM_BTREE);
+                } else {
+                    searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+                            : new PrimaryIndexSearchOperationCallbackFactory(jobId, datasetId, primaryKeyFields,
+                                    txnSubsystemProvider, ResourceType.LSM_BTREE);
+                }
+            }
+            Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(
+                    dataset, mdTxnCtx);
+            AsterixRuntimeComponentsProvider rtcProvider = AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER;
+            BTreeSearchOperatorDescriptor btreeSearchOp;
+            if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+                btreeSearchOp = new BTreeSearchOperatorDescriptor(jobSpec, outputRecDesc,
+                        appContext.getStorageManagerInterface(), appContext.getIndexLifecycleManagerProvider(),
+                        spPc.first, typeTraits, comparatorFactories, bloomFilterKeyFields, lowKeyFields, highKeyFields,
+                        lowKeyInclusive, highKeyInclusive, new LSMBTreeDataflowHelperFactory(
+                                new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), compactionInfo.first,
+                                compactionInfo.second, isSecondary ? new SecondaryIndexOperationTrackerProvider(
+                                        dataset.getDatasetId()) : new PrimaryIndexOperationTrackerProvider(
+                                        dataset.getDatasetId()), rtcProvider,
+                                LSMBTreeIOOperationCallbackFactory.INSTANCE,
+                                storageProperties.getBloomFilterFalsePositiveRate(), !isSecondary, filterTypeTraits,
+                                filterCmpFactories, btreeFields, filterFields, !temp), retainInput, retainNull,
+                        context.getNullWriterFactory(), searchCallbackFactory, minFilterFieldIndexes,
+                        maxFilterFieldIndexes);
+            } else {
+                // External dataset <- use the btree with buddy btree->
+                // Be Careful of Key Start Index ?
+                int[] buddyBreeFields = new int[] { numSecondaryKeys };
+                ExternalBTreeWithBuddyDataflowHelperFactory indexDataflowHelperFactory = new ExternalBTreeWithBuddyDataflowHelperFactory(
+                        compactionInfo.first, compactionInfo.second, new SecondaryIndexOperationTrackerProvider(
+                                dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                        LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE, getStorageProperties()
+                                .getBloomFilterFalsePositiveRate(), buddyBreeFields,
+                        ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, this), !temp);
+                btreeSearchOp = new ExternalBTreeSearchOperatorDescriptor(jobSpec, outputRecDesc, rtcProvider,
+                        rtcProvider, spPc.first, typeTraits, comparatorFactories, bloomFilterKeyFields, lowKeyFields,
+                        highKeyFields, lowKeyInclusive, highKeyInclusive, indexDataflowHelperFactory, retainInput,
+                        retainNull, context.getNullWriterFactory(), searchCallbackFactory);
+            }
+
+            return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeSearchOp, spPc.second);
+
+        } catch (MetadataException me) {
+            throw new AlgebricksException(me);
+        }
+    }
+
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildBtreeRuntime(JobSpecification jobSpec,
+            List<LogicalVariable> outputVars, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv,
+            JobGenContext context, boolean retainInput, boolean retainNull, Dataset dataset, String indexName,
+            int[] lowKeyFields, int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive,
             Object implConfig, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes) throws AlgebricksException {
 
         boolean isSecondary = true;
@@ -1240,6 +1387,109 @@
         }
     }
 
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertOrDeleteRuntime(IndexOperation indexOp,
+            IDataSource<AqlSourceId> dataSource, IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv,
+            List<LogicalVariable> keys, LogicalVariable payload, List<LogicalVariable> additionalNonKeyFields,
+            RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec, boolean bulkload, int nodeId)
+            throws AlgebricksException {
+
+        String datasetName = dataSource.getId().getDatasourceName();
+        Dataset dataset = findDataset(dataSource.getId().getDataverseName(), datasetName);
+        if (dataset == null) {
+            throw new AlgebricksException("Unknown dataset " + datasetName + " in dataverse "
+                    + dataSource.getId().getDataverseName());
+        }
+        boolean temp = dataset.getDatasetDetails().isTemp();
+        isTemporaryDatasetWriteJob = isTemporaryDatasetWriteJob && temp;
+
+        int numKeys = keys.size();
+        int numFilterFields = DatasetUtils.getFilterField(dataset) == null ? 0 : 1;
+        // Move key fields to front.
+        int[] fieldPermutation = new int[numKeys + 1 + numFilterFields];
+        int[] bloomFilterKeyFields = new int[numKeys];
+        int i = 0;
+        for (LogicalVariable varKey : keys) {
+            int idx = propagatedSchema.findVariable(varKey);
+            fieldPermutation[i] = idx;
+            bloomFilterKeyFields[i] = i;
+            i++;
+        }
+        fieldPermutation[numKeys] = propagatedSchema.findVariable(payload);
+        if (numFilterFields > 0) {
+            int idx = propagatedSchema.findVariable(additionalNonKeyFields.get(0));
+            fieldPermutation[numKeys + 1] = idx;
+        }
+
+        try {
+            Index primaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
+                    dataset.getDatasetName(), dataset.getDatasetName());
+            String indexName = primaryIndex.getIndexName();
+
+            String itemTypeName = dataset.getItemTypeName();
+            ARecordType itemType = (ARecordType) MetadataManager.INSTANCE.getDatatype(mdTxnCtx,
+                    dataSource.getId().getDataverseName(), itemTypeName).getDatatype();
+
+            ITypeTraits[] typeTraits = DatasetUtils.computeTupleTypeTraits(dataset, itemType);
+
+            IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
+            IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
+                    itemType, context.getBinaryComparatorFactoryProvider());
+
+            Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = null;
+            if (-1 != nodeId)
+                splitsAndConstraint = splitProviderAndPartitionConstraintsForDataset(dataSource.getId()
+                        .getDataverseName(), datasetName, indexName, temp, nodeId);
+            else
+                splitsAndConstraint = splitProviderAndPartitionConstraintsForDataset(dataSource.getId()
+                        .getDataverseName(), datasetName, indexName, temp);
+
+            // prepare callback
+            JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
+            int datasetId = dataset.getDatasetId();
+            int[] primaryKeyFields = new int[numKeys];
+            for (i = 0; i < numKeys; i++) {
+                primaryKeyFields[i] = i;
+            }
+
+            ITypeTraits[] filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, itemType);
+            IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(
+                    dataset, itemType, context.getBinaryComparatorFactoryProvider());
+            int[] filterFields = DatasetUtils.createFilterFields(dataset);
+            int[] btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset);
+
+            TransactionSubsystemProvider txnSubsystemProvider = new TransactionSubsystemProvider();
+            IModificationOperationCallbackFactory modificationCallbackFactory = temp ? new TempDatasetPrimaryIndexModificationOperationCallbackFactory(
+                    jobId, datasetId, primaryKeyFields, txnSubsystemProvider, indexOp, ResourceType.LSM_BTREE)
+                    : new PrimaryIndexModificationOperationCallbackFactory(jobId, datasetId, primaryKeyFields,
+                            txnSubsystemProvider, indexOp, ResourceType.LSM_BTREE);
+
+            Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(
+                    dataset, mdTxnCtx);
+            IIndexDataflowHelperFactory idfh = new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
+                    datasetId), compactionInfo.first, compactionInfo.second, new PrimaryIndexOperationTrackerProvider(
+                    dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
+                    true, filterTypeTraits, filterCmpFactories, btreeFields, filterFields, !temp);
+            IOperatorDescriptor op;
+            if (bulkload) {
+                long numElementsHint = getCardinalityPerPartitionHint(dataset);
+                op = new TreeIndexBulkLoadOperatorDescriptor(spec, recordDesc, appContext.getStorageManagerInterface(),
+                        appContext.getIndexLifecycleManagerProvider(), splitsAndConstraint.first, typeTraits,
+                        comparatorFactories, bloomFilterKeyFields, fieldPermutation,
+                        GlobalConfig.DEFAULT_TREE_FILL_FACTOR, true, numElementsHint, true, idfh);
+            } else {
+                op = new AsterixLSMTreeInsertDeleteOperatorDescriptor(spec, recordDesc,
+                        appContext.getStorageManagerInterface(), appContext.getIndexLifecycleManagerProvider(),
+                        splitsAndConstraint.first, typeTraits, comparatorFactories, bloomFilterKeyFields,
+                        fieldPermutation, indexOp, idfh, null, modificationCallbackFactory, true, indexName);
+            }
+            return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(op, splitsAndConstraint.second);
+
+        } catch (MetadataException me) {
+            throw new AlgebricksException(me);
+        }
+    }
+
     @Override
     public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(
             IDataSource<AqlSourceId> dataSource, IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv,
@@ -1257,6 +1507,16 @@
             RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec) throws AlgebricksException {
         return getInsertOrDeleteRuntime(IndexOperation.DELETE, dataSource, propagatedSchema, typeEnv, keys, payload,
                 additionalNonKeyFields, recordDesc, context, spec, false);
+    }
+
+    @Override
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(
+            IDataSource<AqlSourceId> dataSource, IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv,
+            List<LogicalVariable> keys, LogicalVariable payload, List<LogicalVariable> additionalNonKeyFields,
+            RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec, int nodeId)
+            throws AlgebricksException {
+        return getInsertOrDeleteRuntime(IndexOperation.DELETE, dataSource, propagatedSchema, typeEnv, keys, payload,
+                additionalNonKeyFields, recordDesc, context, spec, false, nodeId);
     }
 
     public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertOrDeleteRuntime(
@@ -2122,6 +2382,24 @@
     }
 
     public Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraintsForDataset(
+            String dataverseName, String datasetName, String targetIdxName, boolean temp, int nodeId)
+            throws AlgebricksException, MetadataException {
+        FileSplit[] splits = splitsForDataset(mdTxnCtx, dataverseName, datasetName, targetIdxName, temp);
+        Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
+        if (MetadataConstants.METADATA_NODEGROUP_NAME.compareTo(dataset.getNodeGroupName()) == 0) {
+            return splitProviderAndPartitionConstraints(splits);
+        }
+        FileSplit[] actual = null;
+        if (nodeId == -1)
+            return splitProviderAndPartitionConstraints(splits);
+        else {
+            actual = new FileSplit[1];
+            actual[0] = splits[nodeId];
+            return splitProviderAndPartitionConstraints(actual);
+        }
+    }
+
+    public Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraintsForDataset(
             String dataverseName, String datasetName, String targetIdxName, boolean temp) throws AlgebricksException {
         FileSplit[] splits = splitsForDataset(mdTxnCtx, dataverseName, datasetName, targetIdxName, temp);
         return splitProviderAndPartitionConstraints(splits);
diff --git a/pom.xml b/pom.xml
index 8d11476..0631cfc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -56,8 +56,10 @@
         <global.test.includes>**/*TestSuite.java,**/*Test.java,${execution.tests}</global.test.includes>
         <global.test.excludes>${optimizer.tests},${metadata.tests},${invalid.tests},${repeated.tests}</global.test.excludes>
     <!-- Versions under dependencymanagement or used in many projects via properties -->
-        <algebricks.version>0.2.16-incubating</algebricks.version>
-        <hyracks.version>0.2.16-incubating</hyracks.version>
+        <!--algebricks.version>0.2.16-incubating</algebricks.version-->
+        <algebricks.version>0.2.17-SNAPSHOT</algebricks.version>
+        <!--hyracks.version>0.2.16-incubating</hyracks.version-->
+        <hyracks.version>0.2.17-SNAPSHOT</hyracks.version>
         <hadoop.version>2.2.0</hadoop.version>
         <junit.version>4.11</junit.version>
         <commons.io.version>2.4</commons.io.version>

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/428
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Wenhai Li <lw...@yahoo.com>

Change in asterixdb[master]: Equivalence induced operations and the testCases

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Equivalence induced operations and the testCases
......................................................................


Patch Set 21:

Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/911/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/428
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
Gerrit-PatchSet: 21
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Wenhai Li <lw...@yahoo.com>
Gerrit-Reviewer: Ildar Absalyamov <il...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Steven Jacobs <sj...@ucr.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Wenhai Li <lw...@yahoo.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Exchange optimization for lookup and delete

Posted by "Wenhai Li (Code Review)" <do...@asterixdb.incubator.apache.org>.

Wenhai Li has uploaded a new patch set (#2).

Change subject: Exchange optimization for lookup and delete
......................................................................

Exchange optimization for lookup and delete

The following commits from your working branch will be included:
exchange optimization for lookup and delete when the search condition is merely composed of the single primary key.

commit 68704b9dd1afc477929339d1b453fbb2f548804d
Author: Michael <lw...@yahoo.com>
Date:   Fri Oct 2 00:01:16 2015 -0700

    exchange optimization for lookup and delete

Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
---
M asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
M pom.xml
3 files changed, 406 insertions(+), 7 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/28/428/2
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/428
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Wenhai Li <lw...@yahoo.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>

Change in asterixdb[master]: Equivalence induced operations and the testCases The followi...

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Equivalence induced operations and the testCases The following commits from your working branch will be included: exchange optimization for lookup and delete when the search condition is merely composed of the single primary key.
......................................................................


Patch Set 12:

Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/762/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/428
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
Gerrit-PatchSet: 12
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Wenhai Li <lw...@yahoo.com>
Gerrit-Reviewer: Ildar Absalyamov <il...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Jianfeng Jia <ji...@gmail.com>
Gerrit-Reviewer: Steven Jacobs <sj...@ucr.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Equivalence induced operations and the testCases The followi...

Posted by "Wenhai Li (Code Review)" <do...@asterixdb.incubator.apache.org>.

Hello Jenkins,

I'd like you to reexamine a change.  Please visit

    https://asterix-gerrit.ics.uci.edu/428

to look at the new patch set (#11).

Change subject: Equivalence induced operations and the testCases The following commits from your working branch will be included: exchange optimization for lookup and delete when the search condition is merely composed of the single primary key.
......................................................................

Equivalence induced operations and the testCases
The following commits from your working branch will be included:
exchange optimization for lookup and delete when the search condition is merely composed of the single primary key.

commit 68704b9dd1afc477929339d1b453fbb2f548804d
Author: Michael <lw...@yahoo.com>
Date:   Fri Oct 2 00:01:16 2015 -0700

    exchange optimization for lookup and delete

Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
---
M asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AnalysisUtil.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/AddEquivalenceClassForRecordConstructorRule.java
A asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PreComputePartitionIdForAtomicOperationRule.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
M asterix-app/src/test/resources/optimizerts/results/query_issue849-2.plan
M asterix-app/src/test/resources/optimizerts/results/query_issue849.plan
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
9 files changed, 1,489 insertions(+), 330 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/28/428/11
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/428
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
Gerrit-PatchSet: 11
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Wenhai Li <lw...@yahoo.com>
Gerrit-Reviewer: Ildar Absalyamov <il...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Jianfeng Jia <ji...@gmail.com>
Gerrit-Reviewer: Steven Jacobs <sj...@ucr.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>

Change in asterixdb[master]: Equivalence induced operations and the testCases

Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.

Jenkins has posted comments on this change.

Change subject: Equivalence induced operations and the testCases
......................................................................


Patch Set 23:

Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-topic/916/

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/428
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
Gerrit-PatchSet: 23
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Wenhai Li <lw...@yahoo.com>
Gerrit-Reviewer: Ildar Absalyamov <il...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Steven Jacobs <sj...@ucr.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Wenhai Li <lw...@yahoo.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No

Change in asterixdb[master]: Equivalence induced operations and the testCases

Posted by "Wenhai Li (Code Review)" <do...@asterixdb.incubator.apache.org>.

Hello Steven Jacobs, Jenkins,

I'd like you to reexamine a change.  Please visit

    https://asterix-gerrit.ics.uci.edu/428

to look at the new patch set (#29).

Change subject: Equivalence induced operations and the testCases
......................................................................

Equivalence induced operations and the testCases

exchange optimization for lookup and delete when the search condition is merely composed of the single primary key.

Merge current master onto the branch
Setup the restricted NodeDomain information in case the query condition explicitly/implicitly contains a single primary key.
1. Cover the insert/delete/upsert/query operations in both structural and semi-structural schemas.
2. Fixed the partition constraint bug in ASTERIXDB-1016
3. Temporarily enforce the secondary delete/insert node selection derived from the primary key, which will be removed once
the AqlMetadataProvider disable the private constraints of secondary operations.
Merge current master onto the branch

Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
---
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/IndexSearchPOperator.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AnalysisUtil.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/AddEquivalenceClassForRecordConstructorRule.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceRandomPartitioningFeedComputationRule.java
A asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PreComputePartitionIdForRoutableOperationRule.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
A asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/ExpressionUtil.java
M asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849-2.plan
M asterixdb/asterix-app/src/test/resources/optimizerts/results/query_issue849.plan
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteUpsertOperator.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
A hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableExpressionListVisitor.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/BulkloadPOperator.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeleteUpsertPOperator.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InsertDeleteUpsertPOperator.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/INodeDomain.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/ResultSetDomain.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/StructuralPropertiesVector.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java
M hyracks-fullstack/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
M hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
34 files changed, 2,071 insertions(+), 428 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/28/428/29
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/428
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
Gerrit-PatchSet: 29
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Wenhai Li <lw...@yahoo.com>
Gerrit-Reviewer: Ildar Absalyamov <il...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Steven Jacobs <sj...@ucr.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Wenhai Li <lw...@yahoo.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>

Change in asterixdb[master]: Equivalence induced operations and the testCases The followi...

Posted by "Steven Jacobs (Code Review)" <do...@asterixdb.incubator.apache.org>.

Steven Jacobs has posted comments on this change.

Change subject: Equivalence induced operations and the testCases The following commits from your working branch will be included: exchange optimization for lookup and delete when the search condition is merely composed of the single primary key.
......................................................................


Patch Set 13:

(6 comments)

One thing that isn't clear for me: Does this only cover the case where the result comes from a single partition, or can it also handle cases where there is a subset of partitions to be used.

This change needs tests (both optimizer and runtime).

https://asterix-gerrit.ics.uci.edu/#/c/428/13/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
File asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java:

Line 94:         return isSinglePartition;
It seems like you don't need the boolean variable. You can just check here whether partitionId is -1 or not.


https://asterix-gerrit.ics.uci.edu/#/c/428/13/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
File asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java:

Line 696:                     spPc = splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(),
should "curIdx != null"


Line 1148:             if (-1 != nodeId)
nodeId != -1


Line 1978:             if (-1 != nodeId)
nodeId != -1


Line 2639:             if (-1 != nodeId)
nodeId != -1


Line 2965:             if (-1 != nodeId)
nodeId != -1


-- 
To view, visit https://asterix-gerrit.ics.uci.edu/428
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: comment
Gerrit-Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
Gerrit-PatchSet: 13
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Wenhai Li <lw...@yahoo.com>
Gerrit-Reviewer: Ildar Absalyamov <il...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Jianfeng Jia <ji...@gmail.com>
Gerrit-Reviewer: Steven Jacobs <sj...@ucr.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: Yes

Change in asterixdb[master]: Equivalence induced operations and the testCases

Posted by "Wenhai Li (Code Review)" <do...@asterixdb.incubator.apache.org>.

Hello Jenkins,

I'd like you to reexamine a change.  Please visit

    https://asterix-gerrit.ics.uci.edu/428

to look at the new patch set (#8).

Change subject: Equivalence induced operations and the testCases
......................................................................

Equivalence induced operations and the testCases

exchange optimization for lookup and delete 
when the search condition is merely composed 
of the single primary key.

commit 68704b9dd1afc477929339d1b453fbb2f548804d
Author: Michael <lw...@yahoo.com>
Date:   Fri Oct 2 00:01:16 2015 -0700

    exchange optimization for lookup and delete

Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
---
M asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/AddEquivalenceClassForRecordConstructorRule.java
A asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PreComputePartitionIdForInsertAndDeleteRule.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
M asterix-app/src/test/resources/optimizerts/results/query_issue849-2.plan
M asterix-app/src/test/resources/optimizerts/results/query_issue849.plan
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
8 files changed, 1,355 insertions(+), 35 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/28/428/8
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/428
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Wenhai Li <lw...@yahoo.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>

Change in asterixdb[master]: Equivalence induced operations and the testCases

Posted by "Wenhai Li (Code Review)" <do...@asterixdb.incubator.apache.org>.

Hello Steven Jacobs, Jenkins,

I'd like you to reexamine a change.  Please visit

    https://asterix-gerrit.ics.uci.edu/428

to look at the new patch set (#18).

Change subject: Equivalence induced operations and the testCases
......................................................................

Equivalence induced operations and the testCases

Exchange optimization for lookup and delete when the search condition is merely composed of the single primary key.

commit 68704b9dd1afc477929339d1b453fbb2f548804d
Author: Michael <lw...@yahoo.com>
Date:   Fri Oct 2 00:01:16 2015 -0700

    exchange optimization for lookup and delete

Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
---
M asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AnalysisUtil.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/AddEquivalenceClassForRecordConstructorRule.java
A asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PreComputePartitionIdForRoutableOperationRule.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
A asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/ExpressionUtil.java
M asterix-app/src/test/resources/optimizerts/results/query_issue849-2.plan
M asterix-app/src/test/resources/optimizerts/results/query_issue849.plan
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
10 files changed, 1,468 insertions(+), 351 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/28/428/18
-- 
To view, visit https://asterix-gerrit.ics.uci.edu/428
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Iaad567eecc4c91019ecba56cca085acde34d5196
Gerrit-PatchSet: 18
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Wenhai Li <lw...@yahoo.com>
Gerrit-Reviewer: Ildar Absalyamov <il...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Steven Jacobs <sj...@ucr.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Wenhai Li <lw...@yahoo.com>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>