You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2016/03/18 07:37:36 UTC

[03/36] incubator-asterixdb git commit: ASTERIXDB-1109: Fixed deletion of records from open secondary index - Avoided creating calling record constructor in delete pipeline - Fixed the case when multiple open indexes enforce the type of the same fiel

ASTERIXDB-1109: Fixed deletion of records from open secondary index
 - Avoided creating calling record constructor in delete pipeline
 - Fixed the case when multiple open indexes enforce the type of the
  same field & covered it with tests

Change-Id: I41bde91401f67918365de7df19dd2f0de20c73d2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/461
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <bu...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/a05f7182
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/a05f7182
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/a05f7182

Branch: refs/heads/master
Commit: a05f71826a4ab2c0e7148009e0b7e269b15e0a2f
Parents: 4e6a923
Author: Ildar Absalyamov <il...@gmail.com>
Authored: Wed Nov 18 17:19:17 2015 -0800
Committer: Ildar Absalyamov <il...@gmail.com>
Committed: Wed Nov 18 22:23:37 2015 -0800

----------------------------------------------------------------------
 ...aterializationForInsertWithSelfScanRule.java |   1 +
 ...IntroduceSecondaryIndexInsertDeleteRule.java | 366 ++++++++++---------
 .../asterix/translator/CompiledStatements.java  |  99 +++--
 .../asterix/aql/translator/AqlTranslator.java   | 334 ++++++++---------
 .../results/disjunction-to-join-delete-1.plan   |  14 +-
 .../results/disjunction-to-join-delete-2.plan   |  29 +-
 .../results/disjunction-to-join-delete-3.plan   |  26 +-
 .../optimizerts/results/scan-delete-all.plan    |   9 +-
 .../scan-delete-rtree-secondary-index.plan      |  11 +-
 .../optimizerts/results/scan-delete.plan        |  12 +-
 ...ont-skip-primary-index-search-in-delete.plan |  11 +-
 .../skip-ngram-index-search-in-delete.plan      |  12 +-
 .../skip-rtree-index-search-in-delete.plan      |  12 +-
 ...-secondary-btree-index-search-in-delete.plan |  12 +-
 .../skip-word-index-search-in-delete.plan       |  12 +-
 .../enforced-field-name-collision.1.ddl.aql     |  29 --
 .../enforced-field-name-collision.1.ddl.aql     |  29 --
 .../index-type-collision.1.ddl.aql              |  29 ++
 .../index-type-promotion-collision.1.ddl.aql    |  29 ++
 .../record-collision.1.ddl.aql                  |  29 ++
 .../multi-index-composite-key.1.ddl.aql         |  50 +++
 .../multi-index-composite-key.2.update.aql      |  36 ++
 .../multi-index-composite-key.3.ddl.aql         |  33 ++
 .../multi-index-composite-key.4.query.aql       |  38 ++
 .../multi-index-composite-key.5.ddl.aql         |  32 ++
 .../multi-index-composite-key.6.query.aql       |  38 ++
 .../multi-index/multi-index.1.ddl.aql           |  43 +++
 .../multi-index/multi-index.2.update.aql        |  40 ++
 .../multi-index/multi-index.3.ddl.aql           |  24 ++
 .../multi-index/multi-index.4.query.aql         |  31 ++
 .../multi-index/multi-index.5.query.aql         |  31 ++
 .../multi-index/multi-index.6.query.aql         |  30 ++
 .../multi-index-composite-key.1.adm             |   2 +
 .../multi-index-composite-key.2.adm             |   2 +
 .../multi-index/multi-index.1.adm               |   2 +
 .../multi-index/multi-index.2.adm               |   2 +
 .../multi-index/multi-index.3.adm               |   2 +
 .../src/test/resources/runtimets/testsuite.xml  |  32 +-
 pom.xml                                         |   4 +-
 39 files changed, 1019 insertions(+), 558 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a05f7182/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceMaterializationForInsertWithSelfScanRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceMaterializationForInsertWithSelfScanRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceMaterializationForInsertWithSelfScanRule.java
index adce8ce..43ec793 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceMaterializationForInsertWithSelfScanRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceMaterializationForInsertWithSelfScanRule.java
@@ -42,6 +42,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOpe
 import org.apache.hyracks.algebricks.core.algebra.operators.physical.MaterializePOperator;
 import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
 
+// TODO: Reconsider if materialization is needed in delete pipeline
 public class IntroduceMaterializationForInsertWithSelfScanRule implements IAlgebraicRewriteRule {
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a05f7182/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index 6eb3807..1ff97b3 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -22,14 +22,12 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Set;
 import java.util.Stack;
 
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
 import org.apache.asterix.aql.util.FunctionUtils;
 import org.apache.asterix.common.config.DatasetConfig.DatasetType;
 import org.apache.asterix.common.config.DatasetConfig.IndexType;
@@ -53,7 +51,12 @@ import org.apache.asterix.om.types.ATypeTag;
 import org.apache.asterix.om.types.AUnionType;
 import org.apache.asterix.om.types.BuiltinType;
 import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
 import org.apache.asterix.om.util.NonTaggedFormatUtil;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.common.utils.Pair;
 import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -73,6 +76,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogi
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
@@ -82,7 +86,8 @@ import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
 public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
         return false;
     }
 
@@ -100,11 +105,14 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
 
         FunctionIdentifier fid = null;
         /** find the record variable */
-        InsertDeleteOperator insertOp = (InsertDeleteOperator) op1;
-        ILogicalExpression recordExpr = insertOp.getPayloadExpression().getValue();
-        List<LogicalVariable> recordVar = new ArrayList<LogicalVariable>();
+        InsertDeleteOperator insertDeleteOp = (InsertDeleteOperator) op1;
+        ILogicalExpression recordExpr = insertDeleteOp.getPayloadExpression().getValue();
+        LogicalVariable recordVar = null;
+        List<LogicalVariable> usedRecordVars = new ArrayList<>();
         /** assume the payload is always a single variable expression */
-        recordExpr.getUsedVariables(recordVar);
+        recordExpr.getUsedVariables(usedRecordVars);
+        if (usedRecordVars.size() == 1)
+            recordVar = usedRecordVars.get(0);
 
         /**
          * op2 is the assign operator which extract primary keys from the record
@@ -112,7 +120,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
          */
         AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
 
-        if (recordVar.size() == 0) {
+        if (recordVar == null) {
             /**
              * For the case primary key-assignment expressions are constant
              * expressions, find assign op that creates record to be
@@ -135,9 +143,9 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                 }
             }
             AssignOperator assignOp2 = (AssignOperator) op2;
-            recordVar.addAll(assignOp2.getVariables());
+            recordVar = assignOp2.getVariables().get(0);
         }
-        AqlDataSource datasetSource = (AqlDataSource) insertOp.getDataSource();
+        AqlDataSource datasetSource = (AqlDataSource) insertDeleteOp.getDataSource();
         AqlMetadataProvider mp = (AqlMetadataProvider) context.getMetadataProvider();
         String dataverseName = datasetSource.getId().getDataverseName();
         String datasetName = datasetSource.getId().getDatasourceName();
@@ -182,19 +190,6 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
             op0.getInputs().clear();
         }
 
-        // Replicate Operator is applied only when doing the bulk-load.
-        AbstractLogicalOperator replicateOp = null;
-
-        if (secondaryIndexTotalCnt > 1 && insertOp.isBulkload()) {
-            // Split the logical plan into "each secondary index update branch"
-            // to replicate each <PK,RECORD> pair.
-            replicateOp = new ReplicateOperator(secondaryIndexTotalCnt);
-            replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
-            replicateOp.setExecutionMode(ExecutionMode.PARTITIONED);
-            context.computeAndSetTypeEnvironmentForOperator(replicateOp);
-            currentTop = replicateOp;
-        }
-
         // Prepare filtering field information
         List<String> additionalFilteringField = ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterField();
         List<LogicalVariable> additionalFilteringVars = null;
@@ -206,53 +201,70 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
             additionalFilteringVars = new ArrayList<LogicalVariable>();
             additionalFilteringAssignExpressions = new ArrayList<Mutable<ILogicalExpression>>();
             additionalFilteringExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-            prepareVarAndExpression(additionalFilteringField, recType.getFieldNames(), recordVar.get(0),
+            prepareVarAndExpression(additionalFilteringField, recType.getFieldNames(), recordVar,
                     additionalFilteringAssignExpressions, additionalFilteringVars, context);
             additionalFilteringAssign = new AssignOperator(additionalFilteringVars,
                     additionalFilteringAssignExpressions);
             for (LogicalVariable var : additionalFilteringVars) {
-                additionalFilteringExpressions.add(new MutableObject<ILogicalExpression>(
-                        new VariableReferenceExpression(var)));
+                additionalFilteringExpressions
+                        .add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(var)));
             }
         }
+        LogicalVariable enforcedRecordVar = recordVar;
 
-        // Iterate each secondary index and applying Index Update operations.
-        for (Index index : indexes) {
-            List<LogicalVariable> projectVars = new ArrayList<LogicalVariable>();
-            VariableUtilities.getUsedVariables(op1, projectVars);
-            if (!index.isSecondaryIndex()) {
-                continue;
-            }
-            LogicalVariable enforcedRecordVar = recordVar.get(0);
-            hasSecondaryIndex = true;
-            //if the index is enforcing field types
-            if (index.isEnforcingKeyFileds()) {
-                try {
-                    DatasetDataSource ds = (DatasetDataSource) (insertOp.getDataSource());
-                    ARecordType insertRecType = (ARecordType) ds.getSchemaTypes()[ds.getSchemaTypes().length - 1];
-                    LogicalVariable castVar = context.newVar();
-                    ARecordType enforcedType = createEnforcedType(insertRecType, index);
+        if (insertDeleteOp.getOperation() == Kind.INSERT) {
+            try {
+                DatasetDataSource ds = (DatasetDataSource) (insertDeleteOp.getDataSource());
+                ARecordType insertRecType = (ARecordType) ds.getSchemaTypes()[ds.getSchemaTypes().length - 1];
+                LogicalVariable castVar = context.newVar();
+                ARecordType enforcedType = createEnforcedType(insertRecType, indexes);
+                if (!enforcedType.equals(insertRecType)) {
                     //introduce casting to enforced type
                     AbstractFunctionCallExpression castFunc = new ScalarFunctionCallExpression(
                             FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.CAST_RECORD));
 
                     castFunc.getArguments().add(
-                            new MutableObject<ILogicalExpression>(insertOp.getPayloadExpression().getValue()));
+                            new MutableObject<ILogicalExpression>(insertDeleteOp.getPayloadExpression().getValue()));
                     TypeComputerUtilities.setRequiredAndInputTypes(castFunc, enforcedType, insertRecType);
-                    AssignOperator newAssignOperator = new AssignOperator(castVar,
+                    AssignOperator castedRecordAssignOperator = new AssignOperator(castVar,
                             new MutableObject<ILogicalExpression>(castFunc));
-                    newAssignOperator.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
-                    currentTop = newAssignOperator;
-                    //project out casted record
-                    projectVars.add(castVar);
+                    castedRecordAssignOperator.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+                    currentTop = castedRecordAssignOperator;
                     enforcedRecordVar = castVar;
-                    context.computeAndSetTypeEnvironmentForOperator(newAssignOperator);
-                    context.computeAndSetTypeEnvironmentForOperator(currentTop);
                     recType = enforcedType;
-                } catch (AsterixException e) {
-                    throw new AlgebricksException(e);
+                    context.computeAndSetTypeEnvironmentForOperator(castedRecordAssignOperator);
                 }
+            } catch (AsterixException e) {
+                throw new AlgebricksException(e);
+            }
+        }
+        Set<LogicalVariable> projectVars = new HashSet<LogicalVariable>();
+        VariableUtilities.getUsedVariables(op1, projectVars);
+        if (enforcedRecordVar != null)
+            projectVars.add(enforcedRecordVar);
+        ProjectOperator project = new ProjectOperator(new ArrayList<LogicalVariable>(projectVars));
+        project.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+        context.computeAndSetTypeEnvironmentForOperator(project);
+        currentTop = project;
+
+        // Replicate Operator is applied only when doing the bulk-load.
+        AbstractLogicalOperator replicateOp = null;
+        if (secondaryIndexTotalCnt > 1 && insertDeleteOp.isBulkload()) {
+            // Split the logical plan into "each secondary index update branch"
+            // to replicate each <PK,RECORD> pair.
+            replicateOp = new ReplicateOperator(secondaryIndexTotalCnt);
+            replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+            replicateOp.setExecutionMode(ExecutionMode.PARTITIONED);
+            context.computeAndSetTypeEnvironmentForOperator(replicateOp);
+            currentTop = replicateOp;
+        }
+
+        // Iterate each secondary index and applying Index Update operations.
+        for (Index index : indexes) {
+            if (!index.isSecondaryIndex()) {
+                continue;
             }
+            hasSecondaryIndex = true;
 
             List<List<String>> secondaryKeyFields = index.getKeyFieldNames();
             List<IAType> secondaryKeyTypes = index.getKeyFieldTypes();
@@ -266,39 +278,35 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
             }
 
             AssignOperator assign = new AssignOperator(secondaryKeyVars, expressions);
-            ProjectOperator project = new ProjectOperator(projectVars);
 
+            ILogicalOperator filterOrAssignOp = null;
             if (additionalFilteringAssign != null) {
-                additionalFilteringAssign.getInputs().add(new MutableObject<ILogicalOperator>(project));
+                filterOrAssignOp = additionalFilteringAssign;
                 assign.getInputs().add(new MutableObject<ILogicalOperator>(additionalFilteringAssign));
             } else {
-                assign.getInputs().add(new MutableObject<ILogicalOperator>(project));
+                filterOrAssignOp = assign;
             }
 
             // Only apply replicate operator when doing bulk-load
-            if (secondaryIndexTotalCnt > 1 && insertOp.isBulkload())
-                project.getInputs().add(new MutableObject<ILogicalOperator>(replicateOp));
+            if (secondaryIndexTotalCnt > 1 && insertDeleteOp.isBulkload())
+                filterOrAssignOp.getInputs().add(new MutableObject<ILogicalOperator>(replicateOp));
             else
-                project.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
-
-            context.computeAndSetTypeEnvironmentForOperator(project);
+                filterOrAssignOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
 
             if (additionalFilteringAssign != null) {
                 context.computeAndSetTypeEnvironmentForOperator(additionalFilteringAssign);
             }
-
             context.computeAndSetTypeEnvironmentForOperator(assign);
             currentTop = assign;
 
             // BTree, Keyword, or n-gram index case
-            if (index.getIndexType() == IndexType.BTREE
-                    || index.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX
+            if (index.getIndexType() == IndexType.BTREE || index.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX
                     || index.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX
                     || index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
                     || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
                 for (LogicalVariable secondaryKeyVar : secondaryKeyVars) {
-                    secondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
-                            secondaryKeyVar)));
+                    secondaryExpressions.add(
+                            new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
                 }
                 Mutable<ILogicalExpression> filterExpression = createFilterExpression(secondaryKeyVars,
                         context.getOutputTypeEnvironment(currentTop), false);
@@ -306,7 +314,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
 
                 // Introduce the TokenizeOperator only when doing bulk-load,
                 // and index type is keyword or n-gram.
-                if (index.getIndexType() != IndexType.BTREE && insertOp.isBulkload()) {
+                if (index.getIndexType() != IndexType.BTREE && insertDeleteOp.isBulkload()) {
 
                     // Check whether the index is length-partitioned or not.
                     // If partitioned, [input variables to TokenizeOperator,
@@ -326,8 +334,8 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                     List<Mutable<ILogicalExpression>> tokenizeKeyExprs = new ArrayList<Mutable<ILogicalExpression>>();
                     LogicalVariable tokenVar = context.newVar();
                     tokenizeKeyVars.add(tokenVar);
-                    tokenizeKeyExprs.add(new MutableObject<ILogicalExpression>(
-                            new VariableReferenceExpression(tokenVar)));
+                    tokenizeKeyExprs
+                            .add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(tokenVar)));
 
                     // Check the field type of the secondary key.
                     IAType secondaryKeyType = null;
@@ -345,21 +353,22 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                     if (isPartitioned) {
                         LogicalVariable lengthVar = context.newVar();
                         tokenizeKeyVars.add(lengthVar);
-                        tokenizeKeyExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
-                                lengthVar)));
+                        tokenizeKeyExprs
+                                .add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(lengthVar)));
                         varTypes.add(BuiltinType.SHORTWITHOUTTYPEINFO);
                     }
 
                     // TokenizeOperator to tokenize [SK, PK] pairs
                     TokenizeOperator tokenUpdate = new TokenizeOperator(dataSourceIndex,
-                            insertOp.getPrimaryKeyExpressions(), secondaryExpressions, tokenizeKeyVars,
-                            filterExpression, insertOp.getOperation(), insertOp.isBulkload(), isPartitioned, varTypes);
+                            insertDeleteOp.getPrimaryKeyExpressions(), secondaryExpressions, tokenizeKeyVars,
+                            filterExpression, insertDeleteOp.getOperation(), insertDeleteOp.isBulkload(), isPartitioned,
+                            varTypes);
                     tokenUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assign));
                     context.computeAndSetTypeEnvironmentForOperator(tokenUpdate);
 
                     IndexInsertDeleteOperator indexUpdate = new IndexInsertDeleteOperator(dataSourceIndex,
-                            insertOp.getPrimaryKeyExpressions(), tokenizeKeyExprs, filterExpression,
-                            insertOp.getOperation(), insertOp.isBulkload());
+                            insertDeleteOp.getPrimaryKeyExpressions(), tokenizeKeyExprs, filterExpression,
+                            insertDeleteOp.getOperation(), insertDeleteOp.isBulkload());
                     indexUpdate.setAdditionalFilteringExpressions(additionalFilteringExpressions);
                     indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(tokenUpdate));
 
@@ -371,15 +380,15 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                 } else {
                     // When TokenizeOperator is not needed
                     IndexInsertDeleteOperator indexUpdate = new IndexInsertDeleteOperator(dataSourceIndex,
-                            insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
-                            insertOp.getOperation(), insertOp.isBulkload());
+                            insertDeleteOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
+                            insertDeleteOp.getOperation(), insertDeleteOp.isBulkload());
                     indexUpdate.setAdditionalFilteringExpressions(additionalFilteringExpressions);
                     indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
 
                     currentTop = indexUpdate;
                     context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
 
-                    if (insertOp.isBulkload())
+                    if (insertDeleteOp.isBulkload())
                         op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
 
                 }
@@ -397,20 +406,17 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                     keyVarList.add(keyVar);
                     AbstractFunctionCallExpression createMBR = new ScalarFunctionCallExpression(
                             FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.CREATE_MBR));
-                    createMBR.getArguments().add(
-                            new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVars
-                                    .get(0))));
-                    createMBR.getArguments().add(
-                            new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(
-                                    new AInt32(dimension)))));
-                    createMBR.getArguments().add(
-                            new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(
-                                    new AInt32(i)))));
+                    createMBR.getArguments().add(new MutableObject<ILogicalExpression>(
+                            new VariableReferenceExpression(secondaryKeyVars.get(0))));
+                    createMBR.getArguments().add(new MutableObject<ILogicalExpression>(
+                            new ConstantExpression(new AsterixConstantValue(new AInt32(dimension)))));
+                    createMBR.getArguments().add(new MutableObject<ILogicalExpression>(
+                            new ConstantExpression(new AsterixConstantValue(new AInt32(i)))));
                     keyExprList.add(new MutableObject<ILogicalExpression>(createMBR));
                 }
                 for (LogicalVariable secondaryKeyVar : keyVarList) {
-                    secondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
-                            secondaryKeyVar)));
+                    secondaryExpressions.add(
+                            new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
                 }
                 AssignOperator assignCoordinates = new AssignOperator(keyVarList, keyExprList);
                 assignCoordinates.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
@@ -422,14 +428,14 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                         context.getOutputTypeEnvironment(assignCoordinates), forceFilter);
                 AqlIndex dataSourceIndex = new AqlIndex(index, dataverseName, datasetName, mp);
                 IndexInsertDeleteOperator indexUpdate = new IndexInsertDeleteOperator(dataSourceIndex,
-                        insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
-                        insertOp.getOperation(), insertOp.isBulkload());
+                        insertDeleteOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
+                        insertDeleteOp.getOperation(), insertDeleteOp.isBulkload());
                 indexUpdate.setAdditionalFilteringExpressions(additionalFilteringExpressions);
                 indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assignCoordinates));
                 currentTop = indexUpdate;
                 context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
 
-                if (insertOp.isBulkload())
+                if (insertDeleteOp.isBulkload())
                     op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
 
             }
@@ -439,79 +445,107 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
             return false;
         }
 
-        if (!insertOp.isBulkload()) {
+        if (!insertDeleteOp.isBulkload()) {
             op0.getInputs().clear();
             op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
         }
         return true;
     }
 
-    public static ARecordType createEnforcedType(ARecordType initialType, Index index) throws AsterixException,
-            AlgebricksException {
+    // Merges typed index fields with specified recordType, allowing indexed fields to be optional.
+    // I.e. the type { "personId":int32, "name": string, "address" : { "street": string } } with typed indexes on age:int32, address.state:string
+    //      will be merged into type { "personId":int32, "name": string, "age": int32? "address" : { "street": string, "state": string? } }
+    // Used by open indexes to enforce the type of an indexed record
+    public static ARecordType createEnforcedType(ARecordType initialType, List<Index> indexes)
+            throws AsterixException, AlgebricksException {
         ARecordType enforcedType = initialType;
-        for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
-            try {
-                Stack<Pair<ARecordType, String>> nestedTypeStack = new Stack<Pair<ARecordType, String>>();
-                List<String> splits = index.getKeyFieldNames().get(i);
-                ARecordType nestedFieldType = enforcedType;
-                boolean openRecords = false;
-                String bridgeName = nestedFieldType.getTypeName();
-                int j;
-                //Build the stack for the enforced type
-                for (j = 1; j < splits.size(); j++) {
-                    nestedTypeStack.push(new Pair<ARecordType, String>(nestedFieldType, splits.get(j - 1)));
-                    bridgeName = nestedFieldType.getTypeName();
-                    nestedFieldType = (ARecordType) enforcedType.getSubFieldType(splits.subList(0, j));
-                    if (nestedFieldType == null) {
-                        openRecords = true;
-                        break;
+        for (Index index : indexes) {
+            if (!index.isSecondaryIndex() || !index.isEnforcingKeyFileds()) {
+                continue;
+            }
+            for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
+                try {
+                    Stack<Pair<ARecordType, String>> nestedTypeStack = new Stack<Pair<ARecordType, String>>();
+                    List<String> splits = index.getKeyFieldNames().get(i);
+                    ARecordType nestedFieldType = enforcedType;
+                    boolean openRecords = false;
+                    String bridgeName = nestedFieldType.getTypeName();
+                    int j;
+                    //Build the stack for the enforced type
+                    for (j = 1; j < splits.size(); j++) {
+                        nestedTypeStack.push(new Pair<ARecordType, String>(nestedFieldType, splits.get(j - 1)));
+                        bridgeName = nestedFieldType.getTypeName();
+                        nestedFieldType = (ARecordType) enforcedType.getSubFieldType(splits.subList(0, j));
+                        if (nestedFieldType == null) {
+                            openRecords = true;
+                            break;
+                        }
                     }
-                }
-                if (openRecords == true) {
-                    //create the smallest record
-                    enforcedType = new ARecordType(splits.get(splits.size() - 2), new String[] { splits.get(splits
-                            .size() - 1) }, new IAType[] { AUnionType.createNullableType(index.getKeyFieldTypes()
-                            .get(i)) }, true);
-                    //create the open part of the nested field
-                    for (int k = splits.size() - 3; k > (j - 2); k--) {
-                        enforcedType = new ARecordType(splits.get(k), new String[] { splits.get(k + 1) },
-                                new IAType[] { AUnionType.createNullableType(enforcedType) }, true);
+                    if (openRecords == true) {
+                        //create the smallest record
+                        enforcedType = new ARecordType(splits.get(splits.size() - 2),
+                                new String[] { splits.get(splits.size() - 1) },
+                                new IAType[] { AUnionType.createNullableType(index.getKeyFieldTypes().get(i)) }, true);
+                        //create the open part of the nested field
+                        for (int k = splits.size() - 3; k > (j - 2); k--) {
+                            enforcedType = new ARecordType(splits.get(k), new String[] { splits.get(k + 1) },
+                                    new IAType[] { AUnionType.createNullableType(enforcedType) }, true);
+                        }
+                        //Bridge the gap
+                        Pair<ARecordType, String> gapPair = nestedTypeStack.pop();
+                        ARecordType parent = gapPair.first;
+
+                        IAType[] parentFieldTypes = ArrayUtils.addAll(parent.getFieldTypes().clone(),
+                                new IAType[] { AUnionType.createNullableType(enforcedType) });
+                        enforcedType = new ARecordType(bridgeName,
+                                ArrayUtils.addAll(parent.getFieldNames(), enforcedType.getTypeName()), parentFieldTypes,
+                                true);
+
+                    } else {
+                        //Schema is closed all the way to the field
+                        //enforced fields are either null or strongly typed
+                        LinkedHashMap<String, IAType> recordNameTypesMap = new LinkedHashMap<String, IAType>();
+                        for (j = 0; j < nestedFieldType.getFieldNames().length; j++) {
+                            recordNameTypesMap.put(nestedFieldType.getFieldNames()[j],
+                                    nestedFieldType.getFieldTypes()[j]);
+                        }
+                        // if a an enforced field already exists and the type is correct
+                        IAType enforcedFieldType = recordNameTypesMap.get(splits.get(splits.size() - 1));
+                        if (enforcedFieldType != null && enforcedFieldType.getTypeTag() == ATypeTag.UNION
+                                && ((AUnionType) enforcedFieldType).isNullableType())
+                            enforcedFieldType = ((AUnionType) enforcedFieldType).getNullableType();
+                        if (enforcedFieldType != null && !ATypeHierarchy.canPromote(enforcedFieldType.getTypeTag(),
+                                index.getKeyFieldTypes().get(i).getTypeTag()))
+                            throw new AlgebricksException("Cannot enforce field " + index.getKeyFieldNames().get(i)
+                                    + " to have type " + index.getKeyFieldTypes().get(i));
+                        if (enforcedFieldType == null)
+                            recordNameTypesMap.put(splits.get(splits.size() - 1),
+                                    AUnionType.createNullableType(index.getKeyFieldTypes().get(i)));
+                        enforcedType = new ARecordType(nestedFieldType.getTypeName(),
+                                recordNameTypesMap.keySet().toArray(new String[recordNameTypesMap.size()]),
+                                recordNameTypesMap.values().toArray(new IAType[recordNameTypesMap.size()]),
+                                nestedFieldType.isOpen());
                     }
-                    //Bridge the gap
-                    Pair<ARecordType, String> gapPair = nestedTypeStack.pop();
-                    ARecordType parent = gapPair.first;
-
-                    IAType[] parentFieldTypes = ArrayUtils.addAll(parent.getFieldTypes().clone(),
-                            new IAType[] { AUnionType.createNullableType(enforcedType) });
-                    enforcedType = new ARecordType(bridgeName, ArrayUtils.addAll(parent.getFieldNames(),
-                            enforcedType.getTypeName()), parentFieldTypes, true);
 
-                } else {
-                    //Schema is closed all the way to the field
-                    //enforced fields are either null or strongly typed
-                    enforcedType = new ARecordType(nestedFieldType.getTypeName(), ArrayUtils.addAll(
-                            nestedFieldType.getFieldNames(), splits.get(splits.size() - 1)), ArrayUtils.addAll(
-                            nestedFieldType.getFieldTypes(),
-                            AUnionType.createNullableType(index.getKeyFieldTypes().get(i))), nestedFieldType.isOpen());
-                }
-
-                //Create the enforcedtype for the nested fields in the schema, from the ground up
-                if (nestedTypeStack.size() > 0) {
-                    while (!nestedTypeStack.isEmpty()) {
-                        Pair<ARecordType, String> nestedTypePair = nestedTypeStack.pop();
-                        ARecordType nestedRecType = nestedTypePair.first;
-                        IAType[] nestedRecTypeFieldTypes = nestedRecType.getFieldTypes().clone();
-                        nestedRecTypeFieldTypes[nestedRecType.findFieldPosition(nestedTypePair.second)] = enforcedType;
-                        enforcedType = new ARecordType(nestedRecType.getTypeName(), nestedRecType.getFieldNames(),
-                                nestedRecTypeFieldTypes, nestedRecType.isOpen());
+                    //Create the enforcedtype for the nested fields in the schema, from the ground up
+                    if (nestedTypeStack.size() > 0) {
+                        while (!nestedTypeStack.isEmpty()) {
+                            Pair<ARecordType, String> nestedTypePair = nestedTypeStack.pop();
+                            ARecordType nestedRecType = nestedTypePair.first;
+                            IAType[] nestedRecTypeFieldTypes = nestedRecType.getFieldTypes().clone();
+                            nestedRecTypeFieldTypes[nestedRecType
+                                    .findFieldPosition(nestedTypePair.second)] = enforcedType;
+                            enforcedType = new ARecordType(nestedRecType.getTypeName() + "_enforced",
+                                    nestedRecType.getFieldNames(), nestedRecTypeFieldTypes, nestedRecType.isOpen());
+                        }
                     }
-                }
 
-            } catch (AsterixException e) {
-                throw new AlgebricksException("Cannot enforce typed fields "
-                        + StringUtils.join(index.getKeyFieldNames()), e);
-            } catch (IOException e) {
-                throw new AsterixException(e);
+                } catch (AsterixException e) {
+                    throw new AlgebricksException(
+                            "Cannot enforce typed fields " + StringUtils.join(index.getKeyFieldNames()), e);
+                } catch (IOException e) {
+                    throw new AsterixException(e);
+                }
             }
         }
         return enforcedType;
@@ -520,9 +554,9 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
     @SuppressWarnings("unchecked")
     private void prepareVarAndExpression(List<String> field, String[] fieldNames, LogicalVariable recordVar,
             List<Mutable<ILogicalExpression>> expressions, List<LogicalVariable> vars, IOptimizationContext context)
-            throws AlgebricksException {
-        Mutable<ILogicalExpression> varRef = new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
-                recordVar));
+                    throws AlgebricksException {
+        Mutable<ILogicalExpression> varRef = new MutableObject<ILogicalExpression>(
+                new VariableReferenceExpression(recordVar));
         int pos = -1;
         if (field.size() == 1) {
             for (int j = 0; j < fieldNames.length; j++) {
@@ -539,14 +573,14 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                 for (int i = 0; i < field.size(); i++) {
                     fieldList.add(new AString(field.get(i)));
                 }
-                Mutable<ILogicalExpression> fieldRef = new MutableObject<ILogicalExpression>(new ConstantExpression(
-                        new AsterixConstantValue(fieldList)));
+                Mutable<ILogicalExpression> fieldRef = new MutableObject<ILogicalExpression>(
+                        new ConstantExpression(new AsterixConstantValue(fieldList)));
                 //Create an expression for the nested case
                 func = new ScalarFunctionCallExpression(
                         FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_NESTED), varRef, fieldRef);
             } else {
-                Mutable<ILogicalExpression> fieldRef = new MutableObject<ILogicalExpression>(new ConstantExpression(
-                        new AsterixConstantValue(new AString(field.get(0)))));
+                Mutable<ILogicalExpression> fieldRef = new MutableObject<ILogicalExpression>(
+                        new ConstantExpression(new AsterixConstantValue(new AString(field.get(0)))));
                 //Create an expression for the open field case (By name)
                 func = new ScalarFunctionCallExpression(
                         FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME), varRef, fieldRef);
@@ -556,8 +590,8 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
             vars.add(newVar);
         } else {
             // Assumes the indexed field is in the closed portion of the type.
-            Mutable<ILogicalExpression> indexRef = new MutableObject<ILogicalExpression>(new ConstantExpression(
-                    new AsterixConstantValue(new AInt32(pos))));
+            Mutable<ILogicalExpression> indexRef = new MutableObject<ILogicalExpression>(
+                    new ConstantExpression(new AsterixConstantValue(new AInt32(pos))));
             AbstractFunctionCallExpression func = new ScalarFunctionCallExpression(
                     FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX), varRef, indexRef);
             expressions.add(new MutableObject<ILogicalExpression>(func));
@@ -581,8 +615,8 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                     FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.IS_NULL),
                     new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
             ScalarFunctionCallExpression notFuncExpr = new ScalarFunctionCallExpression(
-                    FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.NOT), new MutableObject<ILogicalExpression>(
-                            isNullFuncExpr));
+                    FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.NOT),
+                    new MutableObject<ILogicalExpression>(isNullFuncExpr));
             filterExpressions.add(new MutableObject<ILogicalExpression>(notFuncExpr));
         }
         // No nullable secondary keys.

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a05f7182/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
index 25649ee..26f6bd3 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
@@ -27,13 +27,9 @@ import org.apache.asterix.aql.base.Expression;
 import org.apache.asterix.aql.base.Statement.Kind;
 import org.apache.asterix.aql.expression.CallExpr;
 import org.apache.asterix.aql.expression.FLWOGRExpression;
-import org.apache.asterix.aql.expression.FieldAccessor;
-import org.apache.asterix.aql.expression.FieldBinding;
 import org.apache.asterix.aql.expression.ForClause;
-import org.apache.asterix.aql.expression.Identifier;
 import org.apache.asterix.aql.expression.LiteralExpr;
 import org.apache.asterix.aql.expression.Query;
-import org.apache.asterix.aql.expression.RecordConstructor;
 import org.apache.asterix.aql.expression.VariableExpr;
 import org.apache.asterix.aql.expression.WhereClause;
 import org.apache.asterix.aql.literal.StringLiteral;
@@ -42,8 +38,6 @@ import org.apache.asterix.common.feeds.FeedConnectionRequest;
 import org.apache.asterix.common.functions.FunctionConstants;
 import org.apache.asterix.common.functions.FunctionSignature;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
-import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 
@@ -83,8 +77,8 @@ public class CompiledStatements {
 
     // added by yasser
     public static class CompiledCreateDataverseStatement implements ICompiledStatement {
-        private String dataverseName;
-        private String format;
+        private final String dataverseName;
+        private final String format;
 
         public CompiledCreateDataverseStatement(String dataverseName, String format) {
             this.dataverseName = dataverseName;
@@ -106,7 +100,7 @@ public class CompiledStatements {
     }
 
     public static class CompiledNodeGroupDropStatement implements ICompiledStatement {
-        private String nodeGroupName;
+        private final String nodeGroupName;
 
         public CompiledNodeGroupDropStatement(String nodeGroupName) {
             this.nodeGroupName = nodeGroupName;
@@ -123,9 +117,9 @@ public class CompiledStatements {
     }
 
     public static class CompiledIndexDropStatement implements ICompiledStatement {
-        private String dataverseName;
-        private String datasetName;
-        private String indexName;
+        private final String dataverseName;
+        private final String datasetName;
+        private final String indexName;
 
         public CompiledIndexDropStatement(String dataverseName, String datasetName, String indexName) {
             this.dataverseName = dataverseName;
@@ -152,8 +146,8 @@ public class CompiledStatements {
     }
 
     public static class CompiledDataverseDropStatement implements ICompiledStatement {
-        private String dataverseName;
-        private boolean ifExists;
+        private final String dataverseName;
+        private final boolean ifExists;
 
         public CompiledDataverseDropStatement(String dataverseName, boolean ifExists) {
             this.dataverseName = dataverseName;
@@ -175,7 +169,7 @@ public class CompiledStatements {
     }
 
     public static class CompiledTypeDropStatement implements ICompiledStatement {
-        private String typeName;
+        private final String typeName;
 
         public CompiledTypeDropStatement(String nodeGroupName) {
             this.typeName = nodeGroupName;
@@ -211,7 +205,8 @@ public class CompiledStatements {
         private final int gramLength;
 
         public CompiledCreateIndexStatement(String indexName, String dataverseName, String datasetName,
-                List<List<String>> keyFields, List<IAType> keyTypes, boolean isEnforced, int gramLength, IndexType indexType) {
+                List<List<String>> keyFields, List<IAType> keyTypes, boolean isEnforced, int gramLength,
+                IndexType indexType) {
             this.indexName = indexName;
             this.dataverseName = dataverseName;
             this.datasetName = datasetName;
@@ -222,10 +217,12 @@ public class CompiledStatements {
             this.indexType = indexType;
         }
 
+        @Override
         public String getDatasetName() {
             return datasetName;
         }
 
+        @Override
         public String getDataverseName() {
             return dataverseName;
         }
@@ -261,11 +258,11 @@ public class CompiledStatements {
     }
 
     public static class CompiledLoadFromFileStatement implements ICompiledDmlStatement {
-        private String dataverseName;
-        private String datasetName;
-        private boolean alreadySorted;
-        private String adapter;
-        private Map<String, String> properties;
+        private final String dataverseName;
+        private final String datasetName;
+        private final boolean alreadySorted;
+        private final String adapter;
+        private final Map<String, String> properties;
 
         public CompiledLoadFromFileStatement(String dataverseName, String datasetName, String adapter,
                 Map<String, String> properties, boolean alreadySorted) {
@@ -276,10 +273,12 @@ public class CompiledStatements {
             this.properties = properties;
         }
 
+        @Override
         public String getDataverseName() {
             return dataverseName;
         }
 
+        @Override
         public String getDatasetName() {
             return datasetName;
         }
@@ -315,10 +314,12 @@ public class CompiledStatements {
             this.varCounter = varCounter;
         }
 
+        @Override
         public String getDataverseName() {
             return dataverseName;
         }
 
+        @Override
         public String getDatasetName() {
             return datasetName;
         }
@@ -338,12 +339,12 @@ public class CompiledStatements {
     }
 
     public static class CompiledConnectFeedStatement implements ICompiledDmlStatement {
-        private String dataverseName;
-        private String feedName;
-        private String datasetName;
-        private String policyName;
+        private final String dataverseName;
+        private final String feedName;
+        private final String datasetName;
+        private final String policyName;
         private Query query;
-        private int varCounter;
+        private final int varCounter;
 
         public CompiledConnectFeedStatement(String dataverseName, String feedName, String datasetName,
                 String policyName, Query query, int varCounter) {
@@ -390,7 +391,7 @@ public class CompiledStatements {
             return policyName;
         }
     }
-    
+
     public static class CompiledSubscribeFeedStatement implements ICompiledDmlStatement {
 
         private final FeedConnectionRequest request;
@@ -432,11 +433,10 @@ public class CompiledStatements {
 
     }
 
-
     public static class CompiledDisconnectFeedStatement implements ICompiledDmlStatement {
-        private String dataverseName;
-        private String datasetName;
-        private String feedName;
+        private final String dataverseName;
+        private final String datasetName;
+        private final String feedName;
         private Query query;
         private int varCounter;
 
@@ -476,15 +476,15 @@ public class CompiledStatements {
     }
 
     public static class CompiledDeleteStatement implements ICompiledDmlStatement {
-        private VariableExpr var;
-        private String dataverseName;
-        private String datasetName;
-        private Expression condition;
-        private int varCounter;
-        private AqlMetadataProvider metadataProvider;
+        private final VariableExpr var;
+        private final String dataverseName;
+        private final String datasetName;
+        private final Expression condition;
+        private final int varCounter;
+        private final AqlMetadataProvider metadataProvider;
 
-        public CompiledDeleteStatement(VariableExpr var, String dataverseName, String datasetName,
-                Expression condition, int varCounter, AqlMetadataProvider metadataProvider) {
+        public CompiledDeleteStatement(VariableExpr var, String dataverseName, String datasetName, Expression condition,
+                int varCounter, AqlMetadataProvider metadataProvider) {
             this.var = var;
             this.dataverseName = dataverseName;
             this.datasetName = datasetName;
@@ -529,23 +529,7 @@ public class CompiledStatements {
                 clauseList.add(whereClause);
             }
 
-            Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
-            if (dataset == null) {
-                throw new AlgebricksException("Unknown dataset " + datasetName);
-            }
-            String itemTypeName = dataset.getItemTypeName();
-            IAType itemType = metadataProvider.findType(dataset.getDataverseName(), itemTypeName);
-            ARecordType recType = (ARecordType) itemType;
-            String[] fieldNames = recType.getFieldNames();
-            List<FieldBinding> fieldBindings = new ArrayList<FieldBinding>();
-            for (int i = 0; i < fieldNames.length; i++) {
-                FieldAccessor fa = new FieldAccessor(var, new Identifier(fieldNames[i]));
-                FieldBinding fb = new FieldBinding(new LiteralExpr(new StringLiteral(fieldNames[i])), fa);
-                fieldBindings.add(fb);
-            }
-            RecordConstructor rc = new RecordConstructor(fieldBindings);
-
-            FLWOGRExpression flowgr = new FLWOGRExpression(clauseList, rc);
+            FLWOGRExpression flowgr = new FLWOGRExpression(clauseList, var);
             Query query = new Query();
             query.setBody(flowgr);
             return query;
@@ -592,7 +576,8 @@ public class CompiledStatements {
         private final int gramLength;
 
         public CompiledIndexCompactStatement(String dataverseName, String datasetName, String indexName,
-                List<List<String>> keyFields, List<IAType> keyTypes, boolean isEnforced, int gramLength, IndexType indexType) {
+                List<List<String>> keyFields, List<IAType> keyTypes, boolean isEnforced, int gramLength,
+                IndexType indexType) {
             super(dataverseName, datasetName);
             this.indexName = indexName;
             this.keyFields = keyFields;