You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2016/02/01 09:31:31 UTC
[12/13] incubator-asterixdb git commit: Add Support for Upsert
Operation
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
index e05c338..f8df183 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
@@ -21,32 +21,38 @@ package org.apache.asterix.optimizer.rules;
import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.lang3.mutable.Mutable;
-
import org.apache.asterix.algebra.operators.CommitOperator;
import org.apache.asterix.algebra.operators.physical.CommitPOperator;
import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.declared.DatasetDataSource;
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator.Kind;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class ReplaceSinkOpWithCommitOpRule implements IAlgebraicRewriteRule {
@Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
- // TODO Auto-generated method stub
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
return false;
}
@@ -63,20 +69,43 @@ public class ReplaceSinkOpWithCommitOpRule implements IAlgebraicRewriteRule {
List<Mutable<ILogicalExpression>> primaryKeyExprs = null;
int datasetId = 0;
AbstractLogicalOperator descendantOp = (AbstractLogicalOperator) sinkOperator.getInputs().get(0).getValue();
+ LogicalVariable upsertVar = null;
+ AssignOperator upsertFlagAssign = null;
while (descendantOp != null) {
- if (descendantOp.getOperatorTag() == LogicalOperatorTag.INDEX_INSERT_DELETE) {
- IndexInsertDeleteOperator indexInsertDeleteOperator = (IndexInsertDeleteOperator) descendantOp;
- if (!indexInsertDeleteOperator.isBulkload()) {
- primaryKeyExprs = indexInsertDeleteOperator.getPrimaryKeyExpressions();
- datasetId = ((DatasetDataSource) indexInsertDeleteOperator.getDataSourceIndex().getDataSource())
- .getDataset().getDatasetId();
+ if (descendantOp.getOperatorTag() == LogicalOperatorTag.INDEX_INSERT_DELETE_UPSERT) {
+ IndexInsertDeleteUpsertOperator indexInsertDeleteUpsertOperator = (IndexInsertDeleteUpsertOperator) descendantOp;
+ if (!indexInsertDeleteUpsertOperator.isBulkload()
+ && indexInsertDeleteUpsertOperator.getPrevSecondaryKeyExprs() == null) {
+ primaryKeyExprs = indexInsertDeleteUpsertOperator.getPrimaryKeyExpressions();
+ datasetId = ((DatasetDataSource) indexInsertDeleteUpsertOperator.getDataSourceIndex()
+ .getDataSource()).getDataset().getDatasetId();
break;
}
- } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE) {
- InsertDeleteOperator insertDeleteOperator = (InsertDeleteOperator) descendantOp;
- if (!insertDeleteOperator.isBulkload()) {
- primaryKeyExprs = insertDeleteOperator.getPrimaryKeyExpressions();
- datasetId = ((DatasetDataSource) insertDeleteOperator.getDataSource()).getDataset().getDatasetId();
+ } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE_UPSERT) {
+ InsertDeleteUpsertOperator insertDeleteUpsertOperator = (InsertDeleteUpsertOperator) descendantOp;
+ if (!insertDeleteUpsertOperator.isBulkload()) {
+ primaryKeyExprs = insertDeleteUpsertOperator.getPrimaryKeyExpressions();
+ datasetId = ((DatasetDataSource) insertDeleteUpsertOperator.getDataSource()).getDataset()
+ .getDatasetId();
+ if (insertDeleteUpsertOperator.getOperation() == Kind.UPSERT) {
+ //we need to add a function that checks if previous record was found
+ upsertVar = context.newVar();
+ //introduce casting to enforced type
+ AbstractFunctionCallExpression isNullFunc = new ScalarFunctionCallExpression(
+ FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_NULL));
+ // The first argument is the record
+ isNullFunc.getArguments().add(new MutableObject<ILogicalExpression>(
+ new VariableReferenceExpression(insertDeleteUpsertOperator.getPrevRecordVar())));
+ // AssignOperator puts in the cast var the casted record
+ upsertFlagAssign = new AssignOperator(upsertVar,
+ new MutableObject<ILogicalExpression>(isNullFunc));
+ // Connect the current top of the plan to the cast operator
+ upsertFlagAssign.getInputs()
+ .add(new MutableObject<ILogicalOperator>(sinkOperator.getInputs().get(0).getValue()));
+ sinkOperator.getInputs().clear();
+ sinkOperator.getInputs().add(new MutableObject<ILogicalOperator>(upsertFlagAssign));
+ context.computeAndSetTypeEnvironmentForOperator(upsertFlagAssign);
+ }
break;
}
}
@@ -102,8 +131,8 @@ public class ReplaceSinkOpWithCommitOpRule implements IAlgebraicRewriteRule {
JobId jobId = mp.getJobId();
//create the logical and physical operator
- CommitOperator commitOperator = new CommitOperator(primaryKeyLogicalVars);
- CommitPOperator commitPOperator = new CommitPOperator(jobId, datasetId, primaryKeyLogicalVars);
+ CommitOperator commitOperator = new CommitOperator(primaryKeyLogicalVars, upsertVar);
+ CommitPOperator commitPOperator = new CommitPOperator(jobId, datasetId, primaryKeyLogicalVars, upsertVar);
commitOperator.setPhysicalOperator(commitPOperator);
//create ExtensionOperator and put the commitOperator in it.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
index 85c7478..9a82309 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
@@ -39,9 +39,9 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOper
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
@@ -282,12 +282,12 @@ public class SweepIllegalNonfunctionalFunctions extends AbstractExtractExprRule
}
@Override
- public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void tag) throws AlgebricksException {
+ public Void visitInsertDeleteUpsertOperator(InsertDeleteUpsertOperator op, Void tag) throws AlgebricksException {
return null;
}
@Override
- public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void tag) throws AlgebricksException {
+ public Void visitIndexInsertDeleteUpsertOperator(IndexInsertDeleteUpsertOperator op, Void tag) throws AlgebricksException {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
index 7221972..5dc9f18 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
@@ -327,6 +327,18 @@ public class CompiledStatements {
}
}
+ public static class CompiledUpsertStatement extends CompiledInsertStatement {
+
+ public CompiledUpsertStatement(String dataverseName, String datasetName, Query query, int varCounter) {
+ super(dataverseName, datasetName, query, varCounter);
+ }
+
+ @Override
+ public Kind getKind() {
+ return Kind.UPSERT;
+ }
+ }
+
public static class CompiledConnectFeedStatement implements ICompiledDmlStatement {
private String dataverseName;
private String feedName;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index cd0d21a..b8fe0e1 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -80,6 +80,7 @@ import org.apache.asterix.om.base.AString;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
import org.apache.asterix.om.functions.AsterixFunctionInfo;
+import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.asterix.runtime.formats.FormatUtils;
@@ -117,7 +118,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceSc
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
@@ -238,8 +239,8 @@ class LangExpressionToPlanTranslator
additionalFilteringAssignExpressions);
}
- InsertDeleteOperator insertOp = new InsertDeleteOperator(targetDatasource, payloadRef, varRefsForLoading,
- InsertDeleteOperator.Kind.INSERT, true);
+ InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, payloadRef,
+ varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, true);
insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
if (additionalFilteringAssign != null) {
@@ -344,17 +345,38 @@ class LangExpressionToPlanTranslator
switch (stmt.getKind()) {
case INSERT: {
- InsertDeleteOperator insertOp = new InsertDeleteOperator(targetDatasource, varRef,
- varRefsForLoading, InsertDeleteOperator.Kind.INSERT, false);
+ InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
+ varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false);
insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
insertOp.getInputs().add(new MutableObject<ILogicalOperator>(assign));
leafOperator = new SinkOperator();
leafOperator.getInputs().add(new MutableObject<ILogicalOperator>(insertOp));
break;
}
+ case UPSERT: {
+ InsertDeleteUpsertOperator upsertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
+ varRefsForLoading, InsertDeleteUpsertOperator.Kind.UPSERT, false);
+ upsertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
+ upsertOp.getInputs().add(new MutableObject<ILogicalOperator>(assign));
+ // Create and add a new variable used for representing the original record
+ ARecordType recordType = (ARecordType) targetDatasource.getItemType();
+ upsertOp.setPrevRecordVar(context.newVar());
+ upsertOp.setPrevRecordType(recordType);
+ if (additionalFilteringField != null) {
+ upsertOp.setPrevFilterVar(context.newVar());
+ try {
+ upsertOp.setPrevFilterType(recordType.getFieldType(additionalFilteringField.get(0)));
+ } catch (IOException e) {
+ throw new AlgebricksException("unable to get the type of filter field");
+ }
+ }
+ leafOperator = new SinkOperator();
+ leafOperator.getInputs().add(new MutableObject<ILogicalOperator>(upsertOp));
+ break;
+ }
case DELETE: {
- InsertDeleteOperator deleteOp = new InsertDeleteOperator(targetDatasource, varRef,
- varRefsForLoading, InsertDeleteOperator.Kind.DELETE, false);
+ InsertDeleteUpsertOperator deleteOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
+ varRefsForLoading, InsertDeleteUpsertOperator.Kind.DELETE, false);
deleteOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
deleteOp.getInputs().add(new MutableObject<ILogicalOperator>(assign));
leafOperator = new SinkOperator();
@@ -362,8 +384,8 @@ class LangExpressionToPlanTranslator
break;
}
case CONNECT_FEED: {
- InsertDeleteOperator insertOp = new InsertDeleteOperator(targetDatasource, varRef,
- varRefsForLoading, InsertDeleteOperator.Kind.INSERT, false);
+ InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
+ varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false);
insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
insertOp.getInputs().add(new MutableObject<ILogicalOperator>(assign));
leafOperator = new SinkOperator();
@@ -371,8 +393,8 @@ class LangExpressionToPlanTranslator
break;
}
case SUBSCRIBE_FEED: {
- ILogicalOperator insertOp = new InsertDeleteOperator(targetDatasource, varRef, varRefsForLoading,
- InsertDeleteOperator.Kind.INSERT, false);
+ ILogicalOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
+ varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false);
insertOp.getInputs().add(new MutableObject<ILogicalOperator>(assign));
leafOperator = new SinkOperator();
leafOperator.getInputs().add(new MutableObject<ILogicalOperator>(insertOp));
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/dblp-small/more-dblp-small-id.txt
----------------------------------------------------------------------
diff --git a/asterix-app/data/dblp-small/more-dblp-small-id.txt b/asterix-app/data/dblp-small/more-dblp-small-id.txt
new file mode 100644
index 0000000..c5cfad6
--- /dev/null
+++ b/asterix-app/data/dblp-small/more-dblp-small-id.txt
@@ -0,0 +1,13 @@
+1:books/acm/kim95/AnnevelinkACFHK95:aaaaaaaaaaaaaaaaa:Jurgen Annevelink Rafiul Ahad Amelia Carlson Daniel H. Fishman Michael L. Heytens William Kent:2002-01-03 42-68 1995 Modern Database Systems db/books/collections/kim95.html#AnnevelinkACFHK95
+2:books/acm/kim95/Blakeley95:OQL[C++] Extending C++ with an Object Query Capability.:José A. Blakeley:2002-01-03 69-88 Modern Database Systems db/books/collections/kim95.html#Blakeley95 1995
+3:books/acm/kim95/BreitbartGS95:SQL bbbbbbbbbbbbbbbbbb:Yuri Breitbart Hector Garcia-Molina Abraham Silberschatz:2004-03-08 573-591 Modern Database Systems books/acm/Kim95 db/books/collections/kim95.html#BreitbartGS95 1995
+4:books/acm/kim95/ChristodoulakisK95:ccccccc:Stavros Christodoulakis Leonidas Koveos:2002-01-03 318-337 1995 Modern Database Systems db/books/collections/kim95.html#ChristodoulakisK95
+5:books/acm/kim95/DayalHW95:ddddddddd:Umeshwar Dayal Eric N. Hanson Jennifer Widom:2002-01-03 434-456 1995 Modern Database Systems db/books/collections/kim95.html#DayalHW95
+666:books/acm/kim95/DittrichD95:hhhhhhhhhhhhBMSs SQL Should Do Better A Critique Based on Early Experiences.:Angelika Kotz Dittrich Klaus R. Dittrich:2002-01-03 238-254 1995 Modern Database Systems db/books/collections/kim95.html#DittrichD95
+7:books/acm/kim95/Garcia-MolinaH95:Distributed Databases.:Hector Garcia-Molina Meichun Hsu:2002-01-03 477-493 1995 Modern Database Systems db/books/collections/kim95.html#Garcia-MolinaH95
+101:journals/jacm/GalilHLSW87:An O(n³log n) deterministic and an O(n³) Las Vegs isomorphism test for trivalent graphs.:Zvi Galil Christoph M. Hoffmann Eugene M. Luks Claus-Peter Schnorr Andreas Weber:2003-11-20 513-531 1987 34 J. ACM 3 http //doi.acm.org/10.1145/28869.28870 db/journals/jacm/jacm34.html#GalilHLSW87
+102:conf/focs/GalilHLSW82:An O(n^3 log n) Deterministic and an O(n^3) Probabilistic Isomorphism Test for Trivalent Graphs:Zvi Galil Christoph M. Hoffmann Eugene M. Luks Claus-Peter Schnorr Andreas Weber:2006-04-25 118-125 conf/focs/FOCS23 1982 FOCS db/conf/focs/focs82.html#GalilHLSW82
+103:journals/jacm/GalilT88:An O(n²(m + n log n)log n) min-cost flow algorithm.:Zvi Galil Éva Tardos:2003-11-20 374-386 1988 35 J. ACM 2 http //doi.acm.org/10.1145/42282.214090 db/journals/jacm/jacm35.html#GalilT88
+104:conf/focs/GalilT86:SQL An O(n^2 (m + n log n) log n) Min-Cost Flow Algorithm:Zvi Galil Éva Tardos:2006-04-25 1-9 conf/focs/FOCS27 1986 FOCS db/conf/focs/focs86.html#GalilT86
+105:series/synthesis/2009Weintraub:Jordan Canonical Form Theory and Practice:Steven H. Weintraub:2009-09-06 Jordan Canonical Form Theory and Practice http //dx.doi.org/10.2200/S00218ED1V01Y200908MAS006 http //dx.doi.org/10.2200/S00218ED1V01Y200908MAS006 2009 Synthesis Lectures on Mathematics & Statistics Morgan & Claypool Publishers
+111:series/synthesis/2009Brozos:The Geometry of Walker Manifolds:Miguel Brozos-Vázquez Eduardo García-Río Peter Gilkey Stana Nikcevic Rámon Vázquez-Lorenzo:2009-09-06 The Geometry of Walker Manifolds http //dx.doi.org/10.2200/S00197ED1V01Y200906MAS005 http //dx.doi.org/10.2200/S00197ED1V01Y200906MAS005 2009 Synthesis Lectures on Mathematics & Statistics Morgan & Claypool Publishers
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/more-fbm-with-send-time.adm
----------------------------------------------------------------------
diff --git a/asterix-app/data/more-fbm-with-send-time.adm b/asterix-app/data/more-fbm-with-send-time.adm
new file mode 100644
index 0000000..faf8ff0
--- /dev/null
+++ b/asterix-app/data/more-fbm-with-send-time.adm
@@ -0,0 +1,10 @@
+{"message-id":1,"author-id":34,"in-response-to":2,"sender-location":point("47.16,77.75"),"message":" love sprint its shortcut-menu is awesome:)","send-time":datetime("2012-01-20T10:10:00")}
+{"message-id":3,"author-id":645,"in-response-to":4,"sender-location":point("48.09,81.01"),"message":" like samsung the plan is amazing","send-time":datetime("2012-03-20T10:10:00")}
+{"message-id":5,"author-id":23,"in-response-to":2,"sender-location":point("34.7,90.76"),"message":" love sprint the customization is mind-blowing","send-time":datetime("2012-05-20T10:10:00")}
+{"message-id":7,"author-id":897,"in-response-to":15,"sender-location":point("32.91,85.05"),"message":" dislike sprint the speed is horrible","send-time":datetime("2012-07-20T10:10:00")}
+{"message-id":9,"author-id":65,"in-response-to":12,"sender-location":point("34.45,96.48"),"message":" love verizon its wireless is good","send-time":datetime("2012-09-20T10:10:00")}
+{"message-id":11,"author-id":232,"in-response-to":1,"sender-location":point("38.97,77.49"),"message":" can't stand at&t its plan is terrible","send-time":datetime("2012-11-20T10:10:00")}
+{"message-id":17,"author-id":10,"in-response-to":6,"sender-location":point("42.26,77.76"),"message":" can't stand t-mobile its voicemail-service is OMG:(","send-time":datetime("2012-12-20T10:10:00")}
+{"message-id":13,"author-id":10,"in-response-to":4,"sender-location":point("42.77,78.92"),"message":" dislike iphone the voice-command is bad:(","send-time":datetime("2013-08-20T10:10:00")}
+{"message-id":19,"author-id":9,"in-response-to":12,"sender-location":point("41.33,85.28"),"message":" love at&t its 3G is good:)","send-time":datetime("2013-09-20T10:10:00")}
+{"message-id":15,"author-id":7,"in-response-to":11,"sender-location":point("44.47,67.11"),"message":" like iphone the voicemail-service is awesome","send-time":datetime("2014-01-20T10:10:00")}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/semistructured/tiny01/more-customer.adm
----------------------------------------------------------------------
diff --git a/asterix-app/data/semistructured/tiny01/more-customer.adm b/asterix-app/data/semistructured/tiny01/more-customer.adm
new file mode 100644
index 0000000..7e7770e
--- /dev/null
+++ b/asterix-app/data/semistructured/tiny01/more-customer.adm
@@ -0,0 +1,4 @@
+{ "cid": 112, "name": "Dorie Love", "age": 12, "address": { "number": 2286, "street": "Lake St.", "city": "Los Angeles" }, "interests": {{ "Coffee" }}, "children": [ { "name": "Grady Lave" }, { "name": "Daysi Lave" } ] }
+{ "cid": 94, "name": "Edgardo Dunnegan", "age": 19,"interests": {{ }}, "children": [ { "name": "Lyndia Dunnegan" } ] }
+{ "cid": 619, "name": "Jacques Gaskill", "interests": {{ "Cigars", "Coffee", "Computers", "Wine" }}, "children": [ { "name": "Angelyn Gaskill" }, { "name": "Jeanett Gaskill", "age": 40 }, { "name": "Emelda Gaskill", "age": 34 } ] }
+{ "cid": 92, "name": "Kenny Laychock", "address": { "number": 4790, "street": "Washington St.", "city": "Portland" }, "interests": {{ "Video Games", "Basketball" }}, "children": [ ] }
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/spatial/moreSpatialData.json
----------------------------------------------------------------------
diff --git a/asterix-app/data/spatial/moreSpatialData.json b/asterix-app/data/spatial/moreSpatialData.json
new file mode 100644
index 0000000..3795511
--- /dev/null
+++ b/asterix-app/data/spatial/moreSpatialData.json
@@ -0,0 +1,9 @@
+{"id": 1, "kwds": "aaaaaaa", "line1": line("4.0,7.0 9.0,7.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.2,7.0"), "circle" : circle("1.0,1.0 10.0"), "point": point("4.1,7.0")}
+{"id": 2, "kwds": "factory hosedan", "line1": line("-4.0,2.0 2.0,2.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("2.0,3.0 2.0"), "point": point("4.1,7.0")}
+{"id": 3, "kwds": "bbbbbbb", "line1": line("3.0,0.0 0.0,4.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("5.5,1.0 10.0"), "point": point("43.5083,-79.3007")}
+{"id": 4, "kwds": "enterprisecamp torcamp", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0 2.0,1.0 1.0,0.0"), "poly2": polygon("2.0,1.0 2.0,2.0 3.0,2.0 3.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("77.0,4.0 30.0"), "point": point("43.5083,-79.3007")}
+{"id": 5, "kwds": "cccccccccc", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("100.0,100.0 100.0,400.0 300.0,400.0 300.0,100.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("88.0,1.0 10.0"), "point": point("43.5083,-79.3007")}
+{"id": 67, "kwds": "enterprisecamp torcamp", "line1": line("0.0,5.0 1.0,7.0"), "line2": line("4.0,7.0 2.0,-17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("3.1,1.0 2.9,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,1.0 10.0"), "point": point("43.5083,-79.3007")}
+{"id": 77, "kwds": "enterprisecamp torcamp", "line1": line("0.0,5.0 4.0,7.1"), "line2": line("4.0,7.0 2.0,-17.0"), "poly1": polygon("-5.0,-2.0 -4.0,-1.0 -3.0,-1.0 -2.0,-2.0 -4.0,-4.0 -5.0,-3.0"), "poly2": polygon("3.0,1.0 3.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("3.0,6.0 5.0,7.0"), "circle" : circle("13.0,75.0 1.0"), "point": point("43.5083,-79.3007")}
+{"id": 87, "kwds": "enterprisecamp torcamp", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("-5.0,-2.0 -4.0,-1.0 -3.0,-1.0 -2.0,-2.0 -4.0,-4.0 -5.0,-3.0"), "poly2": polygon("-3.0,-3.0 -1.0,-3.0 -3.0,-5.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("76.0,87.0 50.0"), "point": point("43.5083,-79.3007")}
+{"id": 12, "kwds": "zzzz", "line1": line("-4.0,2.0 2.0,2.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("2.0,3.0 2.0"), "point": point("4.1,7.0")}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/tpch0.001/other-orders.tbl
----------------------------------------------------------------------
diff --git a/asterix-app/data/tpch0.001/other-orders.tbl b/asterix-app/data/tpch0.001/other-orders.tbl
new file mode 100644
index 0000000..933e96a
--- /dev/null
+++ b/asterix-app/data/tpch0.001/other-orders.tbl
@@ -0,0 +1,14 @@
+5954|22|F|167262.34|1992-12-03|1-URGENT|Clerk#000000968|0|requests along the blith|
+5955|99|P|67944.38|1995-03-27|5-LOW|Clerk#000000340|0|deas integrate. fluffily regular pa|
+5956|22|O|118036.54|1998-05-18|1-URGENT|Clerk#000000587|0|le even, express platelets.|
+5957|88|F|230949.45|1993-12-27|2-HIGH|Clerk#000000020|0| dependencies are slyly. bold accounts according to the carefully regular r|
+5958|115|O|145060.41|1995-09-16|3-MEDIUM|Clerk#000000787|0|e final requests detect alongside of the qu|
+5959|23|F|195515.26|1992-05-15|3-MEDIUM|Clerk#000000913|0|into beans use ironic, unusual foxes. carefully regular excuses boost caref|
+5984|70|F|83413.30|1994-06-18|5-LOW|Clerk#000000023|0|ickly final pains haggle along the furiously ironic pinto bea|
+5985|143|F|3942.73|1995-01-12|3-MEDIUM|Clerk#000000417|0|as nag fluffily slyly permanent accounts. regular depo|
+10986|115|F|92187.80|1992-04-22|2-HIGH|Clerk#000000674|0|iously unusual notornis are |
+5986|55|F|92187.80|1992-04-22|2-HIGH|Clerk#000000674|0|iously unusual notornis are |
+10987|64|O|98956.82|1996-08-03|1-URGENT|Clerk#000000464|0| ideas. quietly final accounts haggle blithely pending escapade|
+5987|44|O|98956.82|1996-08-03|1-URGENT|Clerk#000000464|0| ideas. quietly final accounts haggle blithely pending escapade|
+10988|31|F|41655.51|1993-11-22|4-NOT SPECIFIED|Clerk#000000867|0|fully express accounts. final pi|
+5988|11|F|41655.51|1993-11-22|4-NOT SPECIFIED|Clerk#000000867|0|fully express accounts. final pi|
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/upsert/raw-data/more-data.txt
----------------------------------------------------------------------
diff --git a/asterix-app/data/upsert/raw-data/more-data.txt b/asterix-app/data/upsert/raw-data/more-data.txt
new file mode 100644
index 0000000..d77c9c9
--- /dev/null
+++ b/asterix-app/data/upsert/raw-data/more-data.txt
@@ -0,0 +1,4 @@
+2,2,Stephen,155
+4,4,Angela,333
+6,6,Tracy,123
+8,8,George,555
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/upsert/raw-data/overlapping.data
----------------------------------------------------------------------
diff --git a/asterix-app/data/upsert/raw-data/overlapping.data b/asterix-app/data/upsert/raw-data/overlapping.data
new file mode 100644
index 0000000..a013ac1
--- /dev/null
+++ b/asterix-app/data/upsert/raw-data/overlapping.data
@@ -0,0 +1,5 @@
+2,10,Nadia,155
+4,10,Igor,333
+6,10,Silvester,123
+1,11,Cloud,777
+12,44,Smith,987
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/data/upsert/raw-data/test-data.txt
----------------------------------------------------------------------
diff --git a/asterix-app/data/upsert/raw-data/test-data.txt b/asterix-app/data/upsert/raw-data/test-data.txt
new file mode 100644
index 0000000..e1e4c47
--- /dev/null
+++ b/asterix-app/data/upsert/raw-data/test-data.txt
@@ -0,0 +1,4 @@
+1,2,Mohammed,155
+3,4,Kate,333
+5,6,William,123
+7,8,Stanly,555
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
index 3852020..5c76c40 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
@@ -41,8 +41,8 @@ public class UpdateAPIServlet extends RESTAPIServlet {
@Override
protected List<Statement.Kind> getAllowedStatements() {
- Kind[] statementsArray = { Kind.DATAVERSE_DECL, Kind.DELETE, Kind.INSERT, Kind.UPDATE, Kind.DML_CMD_LIST,
- Kind.LOAD, Kind.CONNECT_FEED, Kind.DISCONNECT_FEED, Kind.SET, Kind.COMPACT,
+ Kind[] statementsArray = { Kind.DATAVERSE_DECL, Kind.DELETE, Kind.INSERT, Kind.UPSERT, Kind.UPDATE,
+ Kind.DML_CMD_LIST, Kind.LOAD, Kind.CONNECT_FEED, Kind.DISCONNECT_FEED, Kind.SET, Kind.COMPACT,
Kind.EXTERNAL_DATASET_REFRESH, Kind.RUN };
return Arrays.asList(statementsArray);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 3b6af6d..4adadf9 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -162,6 +162,7 @@ import org.apache.asterix.translator.CompiledStatements.CompiledIndexDropStateme
import org.apache.asterix.translator.CompiledStatements.CompiledInsertStatement;
import org.apache.asterix.translator.CompiledStatements.CompiledLoadFromFileStatement;
import org.apache.asterix.translator.CompiledStatements.CompiledSubscribeFeedStatement;
+import org.apache.asterix.translator.CompiledStatements.CompiledUpsertStatement;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
import org.apache.asterix.translator.TypeTranslator;
import org.apache.asterix.translator.util.ValidateUtil;
@@ -330,8 +331,9 @@ public class QueryTranslator extends AbstractLangTranslator {
handleLoadStatement(metadataProvider, stmt, hcc);
break;
}
- case INSERT: {
- handleInsertStatement(metadataProvider, stmt, hcc);
+ case INSERT:
+ case UPSERT: {
+ handleInsertUpsertStatement(metadataProvider, stmt, hcc);
break;
}
case DELETE: {
@@ -853,7 +855,8 @@ public class QueryTranslator extends AbstractLangTranslator {
fieldType = typeMap.get(typeSignature);
}
if (fieldType == null) {
- throw new AlgebricksException("Unknown type " + fieldExpr.second);
+ throw new AlgebricksException(
+ "Unknown type " + (fieldExpr.second == null ? fieldExpr.first : fieldExpr.second));
}
indexFields.add(fieldExpr.first);
@@ -1817,22 +1820,33 @@ public class QueryTranslator extends AbstractLangTranslator {
}
}
- private void handleInsertStatement(AqlMetadataProvider metadataProvider, Statement stmt,
+ private void handleInsertUpsertStatement(AqlMetadataProvider metadataProvider, Statement stmt,
IHyracksClientConnection hcc) throws Exception {
- InsertStatement stmtInsert = (InsertStatement) stmt;
- String dataverseName = getActiveDataverse(stmtInsert.getDataverseName());
- Query query = stmtInsert.getQuery();
+ InsertStatement stmtInsertUpsert = (InsertStatement) stmt;
+ String dataverseName = getActiveDataverse(stmtInsertUpsert.getDataverseName());
+ Query query = stmtInsertUpsert.getQuery();
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
- MetadataLockManager.INSTANCE.insertDeleteBegin(dataverseName, dataverseName + "." + stmtInsert.getDatasetName(),
- query.getDataverses(), query.getDatasets());
+ MetadataLockManager.INSTANCE.insertDeleteUpsertBegin(dataverseName,
+ dataverseName + "." + stmtInsertUpsert.getDatasetName(), query.getDataverses(), query.getDatasets());
try {
metadataProvider.setWriteTransaction(true);
- CompiledInsertStatement clfrqs = new CompiledInsertStatement(dataverseName,
- stmtInsert.getDatasetName().getValue(), query, stmtInsert.getVarCounter());
+ CompiledInsertStatement clfrqs = null;
+ switch (stmtInsertUpsert.getKind()) {
+ case INSERT:
+ clfrqs = new CompiledInsertStatement(dataverseName, stmtInsertUpsert.getDatasetName().getValue(),
+ query, stmtInsertUpsert.getVarCounter());
+ break;
+ case UPSERT:
+ clfrqs = new CompiledUpsertStatement(dataverseName, stmtInsertUpsert.getDatasetName().getValue(),
+ query, stmtInsertUpsert.getVarCounter());
+ break;
+ default:
+ throw new AlgebricksException("Unsupported statement type " + stmtInsertUpsert.getKind());
+ }
JobSpecification compiled = rewriteCompileQuery(metadataProvider, query, clfrqs);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -1848,8 +1862,9 @@ public class QueryTranslator extends AbstractLangTranslator {
}
throw e;
} finally {
- MetadataLockManager.INSTANCE.insertDeleteEnd(dataverseName,
- dataverseName + "." + stmtInsert.getDatasetName(), query.getDataverses(), query.getDatasets());
+ MetadataLockManager.INSTANCE.insertDeleteUpsertEnd(dataverseName,
+ dataverseName + "." + stmtInsertUpsert.getDatasetName(), query.getDataverses(),
+ query.getDatasets());
}
}
@@ -1861,8 +1876,9 @@ public class QueryTranslator extends AbstractLangTranslator {
MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
boolean bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
- MetadataLockManager.INSTANCE.insertDeleteBegin(dataverseName, dataverseName + "." + stmtDelete.getDatasetName(),
- stmtDelete.getDataverses(), stmtDelete.getDatasets());
+ MetadataLockManager.INSTANCE.insertDeleteUpsertBegin(dataverseName,
+ dataverseName + "." + stmtDelete.getDatasetName(), stmtDelete.getDataverses(),
+ stmtDelete.getDatasets());
try {
metadataProvider.setWriteTransaction(true);
@@ -1884,7 +1900,7 @@ public class QueryTranslator extends AbstractLangTranslator {
}
throw e;
} finally {
- MetadataLockManager.INSTANCE.insertDeleteEnd(dataverseName,
+ MetadataLockManager.INSTANCE.insertDeleteUpsertEnd(dataverseName,
dataverseName + "." + stmtDelete.getDatasetName(), stmtDelete.getDataverses(),
stmtDelete.getDatasets());
}
@@ -2822,7 +2838,8 @@ public class QueryTranslator extends AbstractLangTranslator {
readDataverses.add(dataverseNameFrom);
List<String> readDatasets = new ArrayList<String>();
readDatasets.add(datasetNameFrom);
- MetadataLockManager.INSTANCE.insertDeleteBegin(dataverseNameTo, datasetNameTo, readDataverses, readDatasets);
+ MetadataLockManager.INSTANCE.insertDeleteUpsertBegin(dataverseNameTo, datasetNameTo, readDataverses,
+ readDatasets);
try {
prepareRunExternalRuntime(metadataProvider, hcc, pregelixStmt, dataverseNameFrom, dataverseNameTo,
datasetNameFrom, datasetNameTo, mdTxnCtx);
@@ -2864,7 +2881,8 @@ public class QueryTranslator extends AbstractLangTranslator {
}
throw e;
} finally {
- MetadataLockManager.INSTANCE.insertDeleteEnd(dataverseNameTo, datasetNameTo, readDataverses, readDatasets);
+ MetadataLockManager.INSTANCE.insertDeleteUpsertEnd(dataverseNameTo, datasetNameTo, readDataverses,
+ readDatasets);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
index 199ae39..4c79965 100644
--- a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
+++ b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
@@ -56,9 +56,14 @@ public class ExecutionTest {
@BeforeClass
public static void setUp() throws Exception {
- File outdir = new File(PATH_ACTUAL);
- outdir.mkdirs();
- ExecutionTestUtil.setUp();
+ try {
+ File outdir = new File(PATH_ACTUAL);
+ outdir.mkdirs();
+ ExecutionTestUtil.setUp();
+ } catch (Throwable th) {
+ th.printStackTrace();
+ throw th;
+ }
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java b/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
index eefd40e..4c10eb2 100644
--- a/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
+++ b/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
@@ -27,10 +27,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
/**
* Manages a Mini (local VM) HDFS cluster with a configured number of datanodes.
- *
* @author ramangrover29
*/
public class HDFSCluster {
@@ -68,8 +69,7 @@ public class HDFSCluster {
conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
conf.addResource(new Path(basePath + PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
cleanupLocal();
- //this constructor is deprecated in hadoop 2x
- //dfsCluster = new MiniDFSCluster(nameNodePort, conf, numDataNodes, true, true, StartupOption.REGULAR, null);
+ setLoggingLevel(Level.WARN);
MiniDFSCluster.Builder build = new MiniDFSCluster.Builder(conf);
build.nameNodePort(nameNodePort);
build.numDataNodes(numDataNodes);
@@ -79,6 +79,11 @@ public class HDFSCluster {
loadData(basePath);
}
+ private void setLoggingLevel(Level level) {
+ Logger rootLogger = Logger.getRootLogger();
+ rootLogger.setLevel(level);
+ }
+
private void loadData(String localDataRoot) throws IOException {
Path destDir = new Path(HDFS_PATH);
dfs.mkdirs(destDir);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.1.ddl.aql
new file mode 100644
index 0000000..5eea00b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.1.ddl.aql
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test filters with upsert pipeline
+ * Expected Res : Success
+ * Date : 13th Jan 2016
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type FacebookMessageType as closed {
+ message-id: int64,
+ author-id: int64,
+ in-response-to: int64?,
+ sender-location: point?,
+ message: string,
+ send-time: datetime
+}
+
+create dataset FacebookMessages(FacebookMessageType)
+primary key message-id;
+
+create dataset FilteredFacebookMessages(FacebookMessageType)
+primary key message-id with filter on send-time;
+
+create index AutherIdx on FilteredFacebookMessages(author-id);
+create index MessageIdx on FilteredFacebookMessages(message);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.2.update.aql
new file mode 100644
index 0000000..eecf79d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.2.update.aql
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+load dataset FilteredFacebookMessages using localfs
+(("path"="asterix_nc1://data/fbm-with-send-time.adm"),("format"="adm"));
+
+load dataset FacebookMessages using localfs
+(("path"="asterix_nc1://data/more-fbm-with-send-time.adm"),("format"="adm"));
+
+upsert into dataset FilteredFacebookMessages(
+ for $x in dataset FacebookMessages
+ return $x
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.3.query.aql
new file mode 100644
index 0000000..d1b3925
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/filtered-dataset/filtered-dataset.3.query.aql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+for $m in dataset('FilteredFacebookMessages')
+where $m.send-time > datetime("2012-08-20T10:10:00")
+return $m;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.1.ddl.aql
new file mode 100644
index 0000000..70960ed
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.1.ddl.aql
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type MyRecord as closed {
+ id: int64,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+}
+
+
+create dataset UpsertTo(MyRecord)
+ primary key id;
+
+ create dataset UpsertFrom(MyRecord)
+ primary key id;
+
+create index btree_index on UpsertTo(kwds);
+create index rtree_index on UpsertTo(point) type rtree;
+create index inverted_index on UpsertTo(kwds) type keyword;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.2.update.aql
new file mode 100644
index 0000000..9e6086d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.2.update.aql
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+load dataset UpsertTo
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/spatial/spatialData.json"),("format"="adm"));
+
+load dataset UpsertFrom
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/spatial/moreSpatialData.json"),("format"="adm"));
+
+
+upsert into dataset UpsertTo(
+for $x in dataset UpsertFrom
+return $x
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.3.query.aql
new file mode 100644
index 0000000..a785237
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/multiple-secondaries/multiple-secondaries.3.query.aql
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+for $o in dataset('UpsertTo')
+where spatial-intersect($o.point, create-polygon([4.0,1.0,4.0,4.0,12.0,4.0,12.0,1.0]))
+order by $o.id
+return {"id":$o.id}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.1.ddl.aql
new file mode 100644
index 0000000..41cbd5a
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.1.ddl.aql
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type OrderTypetmp as closed {
+ o_orderkey: int64,
+ o_custkey: int64,
+ o_orderstatus: string,
+ o_totalprice: double,
+ o_orderdate: string,
+ o_orderpriority: string,
+ o_clerk: string,
+ o_shippriority: int64,
+ o_comment: string
+}
+
+create type OrderType as closed {
+nested : OrderTypetmp
+}
+
+create dataset UpsertTo(OrderTypetmp)
+ primary key o_orderkey;
+
+create dataset UpsertFrom(OrderTypetmp)
+ primary key o_orderkey;
+
+ create dataset Orders(OrderType)
+ primary key nested.o_orderkey;
+
+create index idx_Orders_Custkey on Orders(nested.o_custkey);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.2.update.aql
new file mode 100644
index 0000000..7a46315
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.2.update.aql
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+load dataset UpsertTo
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/tpch0.001/orders.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
+load dataset UpsertFrom
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/tpch0.001/other-orders.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
+insert into dataset Orders
+(
+ for $c in dataset('UpsertTo')
+ return {
+ "nested" : $c
+ }
+);
+
+upsert into dataset Orders
+(
+ for $c in dataset('UpsertFrom')
+ return {
+ "nested" : $c
+ }
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.3.query.aql
new file mode 100644
index 0000000..6af352c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nested-index/nested-index.3.query.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+for $o in dataset('Orders')
+where
+ $o.nested.o_custkey < 60
+order by $o.nested.o_orderkey
+return {
+ "o_orderkey": $o.nested.o_orderkey,
+ "o_custkey": $o.nested.o_custkey
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.1.ddl.aql
new file mode 100644
index 0000000..c281fe1
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.1.ddl.aql
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type AddressType as open {
+ number: int64,
+ street: string,
+ city: string
+}
+
+create type CustomerType as open {
+ cid: int64,
+ name: string,
+ age: int64?,
+ address: AddressType?,
+ interests: {{string}},
+ children: [ { name: string, age: int64? } ]
+}
+
+create dataset Customers(CustomerType) primary key cid;
+create dataset MoreCustomers(CustomerType) primary key cid;
+create index age_index on Customers(age);
+
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.2.update.aql
new file mode 100644
index 0000000..589eb3f
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.2.update.aql
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+load dataset Customers
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/semistructured/tiny01/customer.adm"),("format"="adm"));
+
+load dataset MoreCustomers
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/semistructured/tiny01/more-customer.adm"),("format"="adm"));
+
+upsert into dataset Customers(
+for $x in dataset MoreCustomers
+return $x
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.3.query.aql
new file mode 100644
index 0000000..747e1ed
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/nullable-index/nullable-index.3.query.aql
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+for $c in dataset('Customers')
+where $c.age < 20
+order by $c.cid
+return $c
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.1.ddl.aql
new file mode 100644
index 0000000..a1755ce
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.1.ddl.aql
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type OrderType as closed {
+ o_orderkey: int64,
+ o_custkey: int64,
+ o_orderstatus: string,
+ o_totalprice: double,
+ o_orderdate: string,
+ o_orderpriority: string,
+ o_clerk: string,
+ o_shippriority: int64,
+ o_comment: string
+}
+
+create type OrderOpenType as open {
+ o_orderkey: int64,
+ o_orderstatus: string,
+ o_totalprice: double,
+ o_orderdate: string,
+ o_orderpriority: string,
+ o_clerk: string,
+ o_shippriority: int64,
+ o_comment: string
+}
+
+create dataset Orders(OrderType)
+ primary key o_orderkey;
+
+ create dataset OtherOrders(OrderType)
+ primary key o_orderkey;
+
+create dataset OrdersOpen(OrderOpenType)
+primary key o_orderkey;
+
+create index idx_Orders_Custkey on OrdersOpen(o_custkey:int32) enforced;
+
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.2.update.aql
new file mode 100644
index 0000000..5951e05
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.2.update.aql
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+load dataset Orders
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/tpch0.001/orders.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
+load dataset OtherOrders
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="asterix_nc1://data/tpch0.001/other-orders.tbl"),("format"="delimited-text"),("delimiter"="|")) pre-sorted;
+
+insert into dataset OrdersOpen (
+ for $x in dataset OtherOrders
+ return $x
+);
+
+upsert into dataset OrdersOpen (
+ for $x in dataset Orders
+ return $x
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.3.query.aql
new file mode 100644
index 0000000..3b12c24
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/open-index/open-index.3.query.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+for $o in dataset('OrdersOpen')
+where
+ $o.o_custkey > 40
+order by $o.o_orderkey
+return {
+ "o_orderkey": $o.o_orderkey,
+ "o_custkey": $o.o_custkey
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.1.ddl.aql
new file mode 100644
index 0000000..79c6fa7
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.1.ddl.aql
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which doesn't have any secondary indexes
+ * Expected Res : Success
+ * Date : Sep 15th 2015
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TestType as closed{
+id:int32,
+age:int32,
+name:string,
+salary:double
+};
+
+create dataset UpsertTo("TestType") primary key id;
+create dataset UpsertFrom("TestType") primary key id;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.2.update.aql
new file mode 100644
index 0000000..4446525
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.2.update.aql
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which doesn't have any secondary indexes
+ * Expected Res : Success
+ * Date : Sep 15th 2015
+ */
+
+use dataverse test;
+// load first dataset
+load dataset UpsertTo using
+localfs(("format"="delimited-text"),
+ ("path"="asterix_nc1://data/upsert/raw-data/overlapping.data"),
+ ("delimiter"=","));
+// load second dataset
+load dataset UpsertFrom using
+localfs(("format"="delimited-text"),
+ ("path"="asterix_nc1://data/upsert/raw-data/test-data.txt,asterix_nc1://data/upsert/raw-data/more-data.txt"),
+ ("delimiter"=","));
+
+// upsert UpsertFrom into UpsertTo
+use dataverse test;
+upsert into dataset UpsertTo(
+ for $x in dataset UpsertFrom
+ return $x
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.3.query.aql
new file mode 100644
index 0000000..d614ed9
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-index/primary-index.3.query.aql
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which doesn't have any secondary indexes
+ * Expected Res : Success
+ * Date : Sep 15th 2015
+ */
+
+use dataverse test;
+for $x in dataset UpsertTo
+return $x;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.1.ddl.aql
new file mode 100644
index 0000000..258230b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.1.ddl.aql
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which has a b-tree secondary index
+ * Expected Res : Success
+ * Date : Sep 15th 2015
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TestType as closed{
+id:int32,
+age:int32,
+name:string,
+salary:double
+};
+
+create dataset UpsertTo("TestType") primary key id;
+create index ageindex on UpsertTo('age');
+create dataset UpsertFrom("TestType") primary key id;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.2.update.aql
new file mode 100644
index 0000000..9207acb
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.2.update.aql
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which has a b-tree secondary index
+ * Expected Res : Success
+ * Date : Sep 15th 2015
+ */
+
+use dataverse test;
+// load first dataset
+load dataset UpsertTo using
+localfs(("format"="delimited-text"),
+ ("path"="asterix_nc1://data/upsert/raw-data/overlapping.data"),
+ ("delimiter"=","));
+// load second dataset
+load dataset UpsertFrom using
+localfs(("format"="delimited-text"),
+ ("path"="asterix_nc1://data/upsert/raw-data/test-data.txt,asterix_nc1://data/upsert/raw-data/more-data.txt"),
+ ("delimiter"=","));
+
+// upsert UpsertFrom into UpsertTo
+use dataverse test;
+upsert into dataset UpsertTo(
+ for $x in dataset UpsertFrom
+ return $x
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.3.query.aql
new file mode 100644
index 0000000..d52feb8
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-btree/primary-secondary-btree.3.query.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Upsert into a dataset which has a b-tree secondary index
+ * Expected Res : Success
+ * Date : Sep 15th 2015
+ */
+
+ // So far this one doesn't use the btree index, need another query
+use dataverse test;
+for $x in dataset UpsertTo
+where $x.age >5
+return $x;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/5dc958ce/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-inverted/primary-secondary-inverted.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-inverted/primary-secondary-inverted.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-inverted/primary-secondary-inverted.1.ddl.aql
new file mode 100644
index 0000000..143511e
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/upsert/primary-secondary-inverted/primary-secondary-inverted.1.ddl.aql
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type DBLPType as closed {
+ id: int64,
+ dblpid: string,
+ title: string,
+ authors: string,
+ misc: string
+}
+
+create dataset UpsertToDBLP(DBLPType)
+ primary key id;
+
+create dataset UpsertFromDBLP(DBLPType)
+ primary key id;
+
+create index keyword_index on UpsertToDBLP(title) type keyword;
\ No newline at end of file