You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2019/04/09 19:35:34 UTC
[asterixdb] branch master updated (0728004 -> 48a7275)
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git.
from 0728004 [NO ISSUE][FUN] Make Dump_Index Function Return An Array of Values
new 79e1ebd [ASTERIXDB-2537][COMP] Incorrect free variables for QuantifiedExpression
new f4c7fe7 [NO ISSUE][RT] Fix Secondary Indexes Bulkload/Upsert
new 01c8ed5 [NO ISSUE] Fix listing of conflicting procs on sample start
new 153b2ce [NO ISSUE][STO] Ensure Invalid Merged Components Are Deleted
new 48a7275 Merge commit '153b2ce970' from 'stabilization-f69489'
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../asterix/app/bootstrap/TestNodeController.java | 2 +-
.../PersistentLocalResourceRepositoryTest.java | 30 ++++++++++++----------
.../sugar-09/sugar-09.1.ddl.sqlpp} | 11 ++++----
.../sugar-09/sugar-09.2.update.sqlpp} | 8 +++---
.../sugar-09/sugar-09.3.query.sqlpp} | 8 +++---
.../results/group-by/sugar-09/sugar-09.3.adm | 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++++
.../ExternalIndexBulkLoadOperatorDescriptor.java | 8 ++++--
.../ExternalIndexBulkLoadOperatorNodePushable.java | 7 ++---
.../ExternalIndexBulkModifyOperatorDescriptor.java | 10 +++++---
...xternalIndexBulkModifyOperatorNodePushable.java | 6 +++--
.../asterix/lang/sqlpp/util/SqlppVariableUtil.java | 11 ++++++++
.../lang/sqlpp/visitor/FreeVariableVisitor.java | 8 ++++--
.../metadata/declared/MetadataProvider.java | 10 ++++----
.../utils/SecondaryIndexOperationsHelper.java | 8 +++---
.../LSMIndexBulkLoadOperatorDescriptor.java | 10 ++++++--
.../LSMIndexBulkLoadOperatorNodePushable.java | 8 +++---
.../LSMSecondaryUpsertOperatorNodePushable.java | 16 +++++++-----
.../src/main/opt/local/bin/start-sample-cluster.sh | 2 +-
.../PersistentLocalResourceRepository.java | 8 +++---
.../IndexBulkLoadOperatorNodePushable.java | 22 +++++++++++++---
.../TreeIndexBulkLoadOperatorDescriptor.java | 13 +++++++++-
22 files changed, 143 insertions(+), 69 deletions(-)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{index-selection/inverted-index-ulist-jaccard/inverted-index-ulist-jaccard.1.ddl.sqlpp => group-by/sugar-09/sugar-09.1.ddl.sqlpp} (90%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{index-selection/cust-index-age-nullable/cust-index-age-nullable.2.update.sqlpp => group-by/sugar-09/sugar-09.2.update.sqlpp} (87%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{aggregate-sql/scalar_count_empty/scalar_count_empty.3.query.sqlpp => group-by/sugar-09/sugar-09.3.query.sqlpp} (82%)
create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/sugar-09/sugar-09.3.adm
[asterixdb] 05/05: Merge commit '153b2ce970' from
'stabilization-f69489'
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 48a72752708891ff6a2bbea0cbe76bebb3788bab
Merge: 0728004 153b2ce
Author: Michael Blow <mb...@apache.org>
AuthorDate: Mon Apr 8 11:53:20 2019 -0400
Merge commit '153b2ce970' from 'stabilization-f69489'
Change-Id: I5b0d2756d839b0bc2e7c59723306921e698a7630
.../asterix/app/bootstrap/TestNodeController.java | 2 +-
.../PersistentLocalResourceRepositoryTest.java | 30 +++++++------
.../group-by/sugar-09/sugar-09.1.ddl.sqlpp | 50 ++++++++++++++++++++++
.../group-by/sugar-09/sugar-09.2.update.sqlpp | 26 +++++++++++
.../group-by/sugar-09/sugar-09.3.query.sqlpp | 27 ++++++++++++
.../results/group-by/sugar-09/sugar-09.3.adm | 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 +++
.../ExternalIndexBulkLoadOperatorDescriptor.java | 8 +++-
.../ExternalIndexBulkLoadOperatorNodePushable.java | 7 +--
.../ExternalIndexBulkModifyOperatorDescriptor.java | 10 +++--
...xternalIndexBulkModifyOperatorNodePushable.java | 6 ++-
.../asterix/lang/sqlpp/util/SqlppVariableUtil.java | 11 +++++
.../lang/sqlpp/visitor/FreeVariableVisitor.java | 8 +++-
.../metadata/declared/MetadataProvider.java | 10 ++---
.../utils/SecondaryIndexOperationsHelper.java | 8 ++--
.../LSMIndexBulkLoadOperatorDescriptor.java | 10 ++++-
.../LSMIndexBulkLoadOperatorNodePushable.java | 8 ++--
.../LSMSecondaryUpsertOperatorNodePushable.java | 16 ++++---
.../src/main/opt/local/bin/start-sample-cluster.sh | 2 +-
.../PersistentLocalResourceRepository.java | 8 ++--
.../IndexBulkLoadOperatorNodePushable.java | 22 ++++++++--
.../TreeIndexBulkLoadOperatorDescriptor.java | 13 +++++-
22 files changed, 232 insertions(+), 56 deletions(-)
diff --cc asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 567b739,4fcf9f8..07da3f9
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@@ -4820,12 -3170,11 +4820,17 @@@
</compilation-unit>
</test-case>
<test-case FilePath="group-by">
+ <compilation-unit name="sugar-08-negative">
+ <output-dir compare="Text">core-01</output-dir>
+ <expected-error>ASX1103: Illegal use of identifier: x</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
+ <compilation-unit name="sugar-09">
+ <output-dir compare="Text">sugar-09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
<compilation-unit name="null">
<output-dir compare="Text">null</output-dir>
</compilation-unit>
diff --cc asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java
index a186cba,28c9b73..3fc5683
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java
@@@ -34,8 -29,9 +34,10 @@@ import org.apache.asterix.lang.common.b
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
+ import org.apache.asterix.lang.common.expression.QuantifiedExpression;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.struct.Identifier;
+ import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.struct.VarIdentifier;
import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause;
import org.apache.asterix.lang.sqlpp.clause.FromClause;
@@@ -192,18 -169,12 +194,27 @@@ public class SqlppVariableUtil
return bindingVars;
}
+ public static List<VariableExpr> getBindingVariables(QuantifiedExpression qe) {
+ List<QuantifiedPair> quantifiedList = qe.getQuantifiedList();
+ List<VariableExpr> bindingVars = new ArrayList<>(quantifiedList.size());
+ for (QuantifiedPair qp : quantifiedList) {
+ bindingVars.add(qp.getVarExpr());
+ }
+ return bindingVars;
+ }
++
+ public static Map<Expression, Identifier> createFieldVariableMap(List<Pair<Expression, Identifier>> fieldList) {
+ Map<Expression, Identifier> fieldVars = new HashMap<>();
+ for (Pair<Expression, Identifier> p : fieldList) {
+ fieldVars.put(p.first, p.second);
+ }
+ return fieldVars;
+ }
+
+ public static void addToFieldVariableList(VariableExpr varExpr, List<Pair<Expression, Identifier>> outFieldList) {
+ VarIdentifier var = varExpr.getVar();
+ VariableExpr newVarExpr = new VariableExpr(var);
+ newVarExpr.setSourceLocation(varExpr.getSourceLocation());
+ outFieldList.add(new Pair<>(newVarExpr, toUserDefinedVariableName(var)));
+ }
}
diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index aaea111,efee1db..318814c
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@@ -356,13 -357,14 +356,13 @@@ public abstract class SecondaryIndexOpe
}
protected LSMIndexBulkLoadOperatorDescriptor createTreeIndexBulkLoadOp(JobSpecification spec,
- int[] fieldPermutation, IIndexDataflowHelperFactory dataflowHelperFactory, float fillFactor)
- throws AlgebricksException {
+ int[] fieldPermutation, IIndexDataflowHelperFactory dataflowHelperFactory, float fillFactor) {
IndexDataflowHelperFactory primaryIndexDataflowHelperFactory = new IndexDataflowHelperFactory(
metadataProvider.getStorageComponentProvider().getStorageManager(), primaryFileSplitProvider);
-
+ // when an index is being created (not loaded) the filtration is introduced in the pipeline -> no tuple filter
LSMIndexBulkLoadOperatorDescriptor treeIndexBulkLoadOp = new LSMIndexBulkLoadOperatorDescriptor(spec,
secondaryRecDesc, fieldPermutation, fillFactor, false, numElementsHint, false, dataflowHelperFactory,
- primaryIndexDataflowHelperFactory, BulkLoadUsage.CREATE_INDEX, dataset.getDatasetId());
+ primaryIndexDataflowHelperFactory, BulkLoadUsage.CREATE_INDEX, dataset.getDatasetId(), null);
treeIndexBulkLoadOp.setSourceLocation(sourceLoc);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, treeIndexBulkLoadOp,
secondaryPartitionConstraint);
@@@ -370,10 -372,11 +370,10 @@@
}
protected TreeIndexBulkLoadOperatorDescriptor createExternalIndexBulkLoadOp(JobSpecification spec,
- int[] fieldPermutation, IIndexDataflowHelperFactory dataflowHelperFactory, float fillFactor)
- throws AlgebricksException {
+ int[] fieldPermutation, IIndexDataflowHelperFactory dataflowHelperFactory, float fillFactor) {
ExternalIndexBulkLoadOperatorDescriptor treeIndexBulkLoadOp = new ExternalIndexBulkLoadOperatorDescriptor(spec,
secondaryRecDesc, fieldPermutation, fillFactor, false, numElementsHint, false, dataflowHelperFactory,
- ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, metadataProvider));
+ ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, metadataProvider), null);
treeIndexBulkLoadOp.setSourceLocation(sourceLoc);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, treeIndexBulkLoadOp,
secondaryPartitionConstraint);
[asterixdb] 01/05: [ASTERIXDB-2537][COMP] Incorrect free variables
for QuantifiedExpression
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 79e1ebd817357a9b492fa5135ea488c41af7b339
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Mon Apr 1 13:40:41 2019 -0700
[ASTERIXDB-2537][COMP] Incorrect free variables for QuantifiedExpression
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Variables bound by a QuantifiedExpression should be excluded
when computing free variables for that expression
Change-Id: If5d0470b0742baa24e3d383bc4aa617a4c6fdd99
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3306
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../group-by/sugar-09/sugar-09.1.ddl.sqlpp | 50 ++++++++++++++++++++++
.../group-by/sugar-09/sugar-09.2.update.sqlpp | 26 +++++++++++
.../group-by/sugar-09/sugar-09.3.query.sqlpp | 27 ++++++++++++
.../results/group-by/sugar-09/sugar-09.3.adm | 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 +++
.../asterix/lang/sqlpp/util/SqlppVariableUtil.java | 10 +++++
.../lang/sqlpp/visitor/FreeVariableVisitor.java | 8 +++-
7 files changed, 125 insertions(+), 2 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/sugar-09/sugar-09.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/sugar-09/sugar-09.1.ddl.sqlpp
new file mode 100644
index 0000000..ff71d84
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/sugar-09/sugar-09.1.ddl.sqlpp
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: Test quantified expression inside SQL aggregate
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type test.AddressType as
+ closed {
+ number : bigint,
+ street : string,
+ city : string
+};
+
+create type test.CustomerType as
+ closed {
+ cid : bigint,
+ name : string,
+ age : bigint?,
+ address : AddressType?,
+ interests : {{string}},
+ children : [{
+ name : string,
+ age : bigint?
+ }
+]
+};
+
+create dataset Customers(CustomerType) primary key cid;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/sugar-09/sugar-09.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/sugar-09/sugar-09.2.update.sqlpp
new file mode 100644
index 0000000..2a47e36
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/sugar-09/sugar-09.2.update.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: Test quantified expression inside SQL aggregate
+ */
+
+use test;
+
+load dataset Customers using localfs ((`path`=`asterix_nc1://data/semistructured/co1k/customer.adm`),(`format`=`adm`));
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/sugar-09/sugar-09.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/sugar-09/sugar-09.3.query.sqlpp
new file mode 100644
index 0000000..1f0af57
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/group-by/sugar-09/sugar-09.3.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: Test quantified expression inside SQL aggregate
+ */
+
+use test;
+
+SELECT VALUE SUM(CASE WHEN (SOME a IN interests SATISFIES a LIKE '%Cooking%' END) THEN 1 ELSE 0 END)
+FROM Customers
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/sugar-09/sugar-09.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/sugar-09/sugar-09.3.adm
new file mode 100644
index 0000000..c147342
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/sugar-09/sugar-09.3.adm
@@ -0,0 +1 @@
+81
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 0031b9f..4fcf9f8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -3170,6 +3170,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="group-by">
+ <compilation-unit name="sugar-09">
+ <output-dir compare="Text">sugar-09</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="group-by">
<compilation-unit name="null">
<output-dir compare="Text">null</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java
index 14dfaeb..28c9b73 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java
@@ -29,7 +29,9 @@ import org.apache.asterix.lang.common.base.ILangExpression;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
+import org.apache.asterix.lang.common.expression.QuantifiedExpression;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.struct.VarIdentifier;
import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause;
import org.apache.asterix.lang.sqlpp.clause.FromClause;
@@ -167,4 +169,12 @@ public class SqlppVariableUtil {
return bindingVars;
}
+ public static List<VariableExpr> getBindingVariables(QuantifiedExpression qe) {
+ List<QuantifiedPair> quantifiedList = qe.getQuantifiedList();
+ List<VariableExpr> bindingVars = new ArrayList<>(quantifiedList.size());
+ for (QuantifiedPair qp : quantifiedList) {
+ bindingVars.add(qp.getVarExpr());
+ }
+ return bindingVars;
+ }
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
index b754533..6945103 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
@@ -373,10 +373,14 @@ public class FreeVariableVisitor extends AbstractSqlppQueryExpressionVisitor<Voi
@Override
public Void visit(QuantifiedExpression qe, Collection<VariableExpr> freeVars) throws CompilationException {
+ Collection<VariableExpr> qeBindingVars = SqlppVariableUtil.getBindingVariables(qe);
+ Collection<VariableExpr> qeFreeVars = new HashSet<>();
for (QuantifiedPair pair : qe.getQuantifiedList()) {
- pair.getExpr().accept(this, freeVars);
+ pair.getExpr().accept(this, qeFreeVars);
}
- qe.getSatisfiesExpr().accept(this, freeVars);
+ qe.getSatisfiesExpr().accept(this, qeFreeVars);
+ qeFreeVars.removeAll(qeBindingVars);
+ freeVars.addAll(qeFreeVars);
return null;
}
[asterixdb] 04/05: [NO ISSUE][STO] Ensure Invalid Merged Components
Are Deleted
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 153b2ce970423d9fa0f02f83951d9b854b4732e6
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Sun Apr 7 21:11:52 2019 +0300
[NO ISSUE][STO] Ensure Invalid Merged Components Are Deleted
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- When cleaning up an index's invalid components, check the
components end sequence in addition to the start sequence
to ensure invalid merged components with are cleaned too.
- Add test case.
Change-Id: I80d72b1b614718e2e283bc72a874c140d178d1e0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3331
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../PersistentLocalResourceRepositoryTest.java | 30 ++++++++++++----------
.../PersistentLocalResourceRepository.java | 8 +++---
2 files changed, 22 insertions(+), 16 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
index 54ae683..00d2d3d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java
@@ -19,6 +19,7 @@
package org.apache.asterix.test.storage;
import java.io.File;
+import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -127,19 +128,9 @@ public class PersistentLocalResourceRepositoryTest {
String invalidComponentRange = invalidComponentId + AbstractLSMIndexFileManager.DELIMITER + invalidComponentId;
FileReference indexDirRef = ncAppCtx.getIoManager().resolve(indexPath);
String indexDir = indexDirRef.getFile().getAbsolutePath();
- // create the invalid component files
- Path btreePath = Paths.get(indexDir, invalidComponentRange + AbstractLSMIndexFileManager.DELIMITER
- + AbstractLSMIndexFileManager.BTREE_SUFFIX);
- Path filterPath = Paths.get(indexDir, invalidComponentRange + AbstractLSMIndexFileManager.DELIMITER
- + AbstractLSMIndexFileManager.BLOOM_FILTER_SUFFIX);
- Files.createFile(btreePath);
- Files.createFile(filterPath);
-
- // clean up the index partition
- localResourceRepository.cleanup(lr.getPartition());
- // ensure that the invalid component was deleted
- Assert.assertFalse(btreePath.toFile().exists());
- Assert.assertFalse(filterPath.toFile().exists());
+ ensureInvalidComponentDeleted(indexDir, invalidComponentRange, localResourceRepository, lr);
+ String invalidMergeComponentRange = "0" + AbstractLSMIndexFileManager.DELIMITER + invalidComponentId;
+ ensureInvalidComponentDeleted(indexDir, invalidMergeComponentRange, localResourceRepository, lr);
// ensure that valid components still exist
// find index valid component timestamp from checkpoint
@@ -179,4 +170,17 @@ public class PersistentLocalResourceRepositoryTest {
Assert.assertFalse(indexMetadataFile.exists());
Assert.assertFalse(indexMetadataMaskFile.exists());
}
+
+ private void ensureInvalidComponentDeleted(String indexDir, String componentSeq,
+ PersistentLocalResourceRepository localResourceRepository, DatasetLocalResource lr) throws IOException {
+ Path btreePath = Paths.get(indexDir,
+ componentSeq + AbstractLSMIndexFileManager.DELIMITER + AbstractLSMIndexFileManager.BTREE_SUFFIX);
+ Path filterPath = Paths.get(indexDir,
+ componentSeq + AbstractLSMIndexFileManager.DELIMITER + AbstractLSMIndexFileManager.BLOOM_FILTER_SUFFIX);
+ Files.createFile(btreePath);
+ Files.createFile(filterPath);
+ localResourceRepository.cleanup(lr.getPartition());
+ Assert.assertFalse(btreePath.toFile().exists());
+ Assert.assertFalse(filterPath.toFile().exists());
+ }
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 8f870c0..aef7bbd 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -497,10 +497,12 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito
}
final long validComponentSequence = getIndexCheckpointManager(index).getValidComponentSequence();
for (File componentFile : indexComponentFiles) {
- // delete any file with start sequence > valid component sequence
+ // delete any file with start or end sequence > valid component sequence
final long fileStart = IndexComponentFileReference.of(componentFile.getName()).getSequenceStart();
- if (fileStart > validComponentSequence) {
- LOGGER.info(() -> "Deleting invalid component file: " + componentFile.getAbsolutePath());
+ final long fileEnd = IndexComponentFileReference.of(componentFile.getName()).getSequenceEnd();
+ if (fileStart > validComponentSequence || fileEnd > validComponentSequence) {
+ LOGGER.warn(() -> "Deleting invalid component file " + componentFile.getAbsolutePath()
+ + " based on valid sequence " + validComponentSequence);
Files.delete(componentFile.toPath());
}
}
[asterixdb] 02/05: [NO ISSUE][RT] Fix Secondary Indexes
Bulkload/Upsert
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit f4c7fe74645debd49f7f082b3e2cc5d15f07f48e
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Wed Apr 3 00:44:35 2019 +0300
[NO ISSUE][RT] Fix Secondary Indexes Bulkload/Upsert
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Prevent null values from being upserted into
secondary indexes.
- Use secondary index tuple filter when bulkloading
to prevent null/missing values from being inserted
into indexes of optional fields.
Change-Id: I7b847aece06a4387cc41389b0566d0c0dc98487d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3304
Reviewed-by: Luo Chen <cl...@uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../asterix/app/bootstrap/TestNodeController.java | 2 +-
.../ExternalIndexBulkLoadOperatorDescriptor.java | 8 ++++++--
.../ExternalIndexBulkLoadOperatorNodePushable.java | 7 ++++---
.../ExternalIndexBulkModifyOperatorDescriptor.java | 10 +++++++---
...xternalIndexBulkModifyOperatorNodePushable.java | 6 ++++--
.../metadata/declared/MetadataProvider.java | 10 +++++-----
.../utils/SecondaryIndexOperationsHelper.java | 8 ++++----
.../LSMIndexBulkLoadOperatorDescriptor.java | 10 ++++++++--
.../LSMIndexBulkLoadOperatorNodePushable.java | 8 ++++----
.../LSMSecondaryUpsertOperatorNodePushable.java | 16 ++++++++-------
.../IndexBulkLoadOperatorNodePushable.java | 23 +++++++++++++++++++---
.../TreeIndexBulkLoadOperatorDescriptor.java | 13 +++++++++++-
12 files changed, 84 insertions(+), 37 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index b07a30e..1a9df0b 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -220,7 +220,7 @@ public class TestNodeController {
LSMIndexBulkLoadOperatorNodePushable op =
new LSMIndexBulkLoadOperatorNodePushable(secondaryIndexHelperFactory, primaryIndexHelperFactory,
ctx, 0, fieldPermutation, 1.0F, false, numElementsHint, true, secondaryIndexInfo.rDesc,
- BulkLoadUsage.CREATE_INDEX, dataset.getDatasetId());
+ BulkLoadUsage.CREATE_INDEX, dataset.getDatasetId(), null);
op.setOutputFrameWriter(0, new SinkRuntimeFactory().createPushRuntime(ctx)[0], null);
return Pair.of(secondaryIndexInfo, op);
} catch (Throwable th) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkLoadOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkLoadOperatorDescriptor.java
index 83e4566..f910fb4 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkLoadOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkLoadOperatorDescriptor.java
@@ -24,6 +24,7 @@ import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
@@ -31,13 +32,16 @@ public class ExternalIndexBulkLoadOperatorDescriptor extends TreeIndexBulkLoadOp
private static final long serialVersionUID = 1L;
private final int version;
+ private final ITupleFilterFactory tupleFilterFactory;
public ExternalIndexBulkLoadOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outRecDesc,
int[] fieldPermutation, float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex, IIndexDataflowHelperFactory indexHelperFactory, int version) {
+ boolean checkIfEmptyIndex, IIndexDataflowHelperFactory indexHelperFactory, int version,
+ ITupleFilterFactory tupleFilterFactory) {
super(spec, outRecDesc, fieldPermutation, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex,
indexHelperFactory);
this.version = version;
+ this.tupleFilterFactory = tupleFilterFactory;
}
@Override
@@ -45,7 +49,7 @@ public class ExternalIndexBulkLoadOperatorDescriptor extends TreeIndexBulkLoadOp
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new ExternalIndexBulkLoadOperatorNodePushable(indexHelperFactory, ctx, partition, fieldPermutation,
fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex,
- recordDescProvider.getInputRecordDescriptor(this.getActivityId(), 0), version);
+ recordDescProvider.getInputRecordDescriptor(this.getActivityId(), 0), version, tupleFilterFactory);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkLoadOperatorNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkLoadOperatorNodePushable.java
index 74bc0dc..39f23ed 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkLoadOperatorNodePushable.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkLoadOperatorNodePushable.java
@@ -25,6 +25,7 @@ import org.apache.asterix.common.ioopcallbacks.LSMIOOperationCallback;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IndexBulkLoadOperatorNodePushable;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
@@ -37,10 +38,10 @@ public class ExternalIndexBulkLoadOperatorNodePushable extends IndexBulkLoadOper
public ExternalIndexBulkLoadOperatorNodePushable(IIndexDataflowHelperFactory indexDataflowHelperFactory,
IHyracksTaskContext ctx, int partition, int[] fieldPermutation, float fillFactor, boolean verifyInput,
- long numElementsHint, boolean checkIfEmptyIndex, RecordDescriptor recDesc, int version)
- throws HyracksDataException {
+ long numElementsHint, boolean checkIfEmptyIndex, RecordDescriptor recDesc, int version,
+ ITupleFilterFactory tupleFilterFactory) throws HyracksDataException {
super(indexDataflowHelperFactory, ctx, partition, fieldPermutation, fillFactor, verifyInput, numElementsHint,
- checkIfEmptyIndex, recDesc);
+ checkIfEmptyIndex, recDesc, tupleFilterFactory);
this.version = version;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorDescriptor.java
index 60f1721..674ee42 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorDescriptor.java
@@ -23,6 +23,7 @@ import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
@@ -30,12 +31,15 @@ public class ExternalIndexBulkModifyOperatorDescriptor extends TreeIndexBulkLoad
private static final long serialVersionUID = 1L;
private final int[] deletedFiles;
+ private final ITupleFilterFactory tupleFilterFactory;
public ExternalIndexBulkModifyOperatorDescriptor(IOperatorDescriptorRegistry spec,
IIndexDataflowHelperFactory dataflowHelperFactory, int[] deletedFiles, int[] fieldPermutation,
- float fillFactor, boolean verifyInput, long numElementsHint) {
- super(spec, null, fieldPermutation, fillFactor, verifyInput, numElementsHint, false, dataflowHelperFactory);
+ float fillFactor, boolean verifyInput, long numElementsHint, ITupleFilterFactory tupleFilterFactory) {
+ super(spec, null, fieldPermutation, fillFactor, verifyInput, numElementsHint, false, dataflowHelperFactory,
+ tupleFilterFactory);
this.deletedFiles = deletedFiles;
+ this.tupleFilterFactory = tupleFilterFactory;
}
@Override
@@ -43,7 +47,7 @@ public class ExternalIndexBulkModifyOperatorDescriptor extends TreeIndexBulkLoad
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new ExternalIndexBulkModifyOperatorNodePushable(indexHelperFactory, ctx, partition, fieldPermutation,
fillFactor, verifyInput, numElementsHint,
- recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), deletedFiles);
+ recordDescProvider.getInputRecordDescriptor(getActivityId(), 0), deletedFiles, tupleFilterFactory);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorNodePushable.java
index 57e2917..745853e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorNodePushable.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorNodePushable.java
@@ -30,6 +30,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IndexBulkLoadOperatorNodePushable;
@@ -45,9 +46,10 @@ public class ExternalIndexBulkModifyOperatorNodePushable extends IndexBulkLoadOp
public ExternalIndexBulkModifyOperatorNodePushable(IIndexDataflowHelperFactory indexHelperFactory,
IHyracksTaskContext ctx, int partition, int[] fieldPermutation, float fillFactor, boolean verifyInput,
- long numElementsHint, RecordDescriptor inputRecDesc, int[] deletedFiles) throws HyracksDataException {
+ long numElementsHint, RecordDescriptor inputRecDesc, int[] deletedFiles,
+ ITupleFilterFactory tupleFilterFactory) throws HyracksDataException {
super(indexHelperFactory, ctx, partition, fieldPermutation, fillFactor, verifyInput, numElementsHint, false,
- inputRecDesc);
+ inputRecDesc, tupleFilterFactory);
this.deletedFiles = deletedFiles;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index e212d11..55bf261 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -642,7 +642,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
new IndexDataflowHelperFactory(storageComponentProvider.getStorageManager(), splitsAndConstraint.first);
LSMIndexBulkLoadOperatorDescriptor btreeBulkLoad = new LSMIndexBulkLoadOperatorDescriptor(spec, null,
fieldPermutation, GlobalConfig.DEFAULT_TREE_FILL_FACTOR, false, numElementsHint, true,
- indexHelperFactory, null, BulkLoadUsage.LOAD, dataset.getDatasetId());
+ indexHelperFactory, null, BulkLoadUsage.LOAD, dataset.getDatasetId(), null);
return new Pair<>(btreeBulkLoad, splitsAndConstraint.second);
}
@@ -1034,7 +1034,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
long numElementsHint = getCardinalityPerPartitionHint(dataset);
op = new LSMIndexBulkLoadOperatorDescriptor(spec, inputRecordDesc, fieldPermutation,
GlobalConfig.DEFAULT_TREE_FILL_FACTOR, true, numElementsHint, true, idfh, null, BulkLoadUsage.LOAD,
- dataset.getDatasetId());
+ dataset.getDatasetId(), null);
} else {
op = new LSMTreeInsertDeleteOperatorDescriptor(spec, inputRecordDesc, fieldPermutation, indexOp, idfh, null,
true, modificationCallbackFactory);
@@ -1159,7 +1159,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
long numElementsHint = getCardinalityPerPartitionHint(dataset);
op = new LSMIndexBulkLoadOperatorDescriptor(spec, inputRecordDesc, fieldPermutation,
GlobalConfig.DEFAULT_TREE_FILL_FACTOR, false, numElementsHint, false, idfh, null,
- BulkLoadUsage.LOAD, dataset.getDatasetId());
+ BulkLoadUsage.LOAD, dataset.getDatasetId(), filterFactory);
} else if (indexOp == IndexOperation.UPSERT) {
int upsertIndicatorFieldIndex = propagatedSchema.findVariable(upsertIndicatorVar);
op = new LSMSecondaryUpsertOperatorDescriptor(spec, inputRecordDesc, fieldPermutation, idfh,
@@ -1259,7 +1259,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
long numElementsHint = getCardinalityPerPartitionHint(dataset);
op = new LSMIndexBulkLoadOperatorDescriptor(spec, recordDesc, fieldPermutation,
GlobalConfig.DEFAULT_TREE_FILL_FACTOR, false, numElementsHint, false, indexDataflowHelperFactory,
- null, BulkLoadUsage.LOAD, dataset.getDatasetId());
+ null, BulkLoadUsage.LOAD, dataset.getDatasetId(), filterFactory);
} else if (indexOp == IndexOperation.UPSERT) {
int upsertIndicatorFieldIndex = propagatedSchema.findVariable(upsertIndicatorVar);
op = new LSMSecondaryUpsertOperatorDescriptor(spec, recordDesc, fieldPermutation,
@@ -1370,7 +1370,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
long numElementsHint = getCardinalityPerPartitionHint(dataset);
op = new LSMIndexBulkLoadOperatorDescriptor(spec, recordDesc, fieldPermutation,
GlobalConfig.DEFAULT_TREE_FILL_FACTOR, false, numElementsHint, false, indexDataFlowFactory,
- null, BulkLoadUsage.LOAD, dataset.getDatasetId());
+ null, BulkLoadUsage.LOAD, dataset.getDatasetId(), filterFactory);
} else if (indexOp == IndexOperation.UPSERT) {
int upsertIndicatorFieldIndex = propagatedSchema.findVariable(upsertIndicatorVar);
op = new LSMSecondaryUpsertOperatorDescriptor(spec, recordDesc, fieldPermutation, indexDataFlowFactory,
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index 5f9e6ef..efee1db 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -361,10 +361,10 @@ public abstract class SecondaryIndexOperationsHelper {
throws AlgebricksException {
IndexDataflowHelperFactory primaryIndexDataflowHelperFactory = new IndexDataflowHelperFactory(
metadataProvider.getStorageComponentProvider().getStorageManager(), primaryFileSplitProvider);
-
+ // when an index is being created (not loaded) the filtration is introduced in the pipeline -> no tuple filter
LSMIndexBulkLoadOperatorDescriptor treeIndexBulkLoadOp = new LSMIndexBulkLoadOperatorDescriptor(spec,
secondaryRecDesc, fieldPermutation, fillFactor, false, numElementsHint, false, dataflowHelperFactory,
- primaryIndexDataflowHelperFactory, BulkLoadUsage.CREATE_INDEX, dataset.getDatasetId());
+ primaryIndexDataflowHelperFactory, BulkLoadUsage.CREATE_INDEX, dataset.getDatasetId(), null);
treeIndexBulkLoadOp.setSourceLocation(sourceLoc);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, treeIndexBulkLoadOp,
secondaryPartitionConstraint);
@@ -376,7 +376,7 @@ public abstract class SecondaryIndexOperationsHelper {
throws AlgebricksException {
ExternalIndexBulkLoadOperatorDescriptor treeIndexBulkLoadOp = new ExternalIndexBulkLoadOperatorDescriptor(spec,
secondaryRecDesc, fieldPermutation, fillFactor, false, numElementsHint, false, dataflowHelperFactory,
- ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, metadataProvider));
+ ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, metadataProvider), null);
treeIndexBulkLoadOp.setSourceLocation(sourceLoc);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, treeIndexBulkLoadOp,
secondaryPartitionConstraint);
@@ -494,7 +494,7 @@ public abstract class SecondaryIndexOperationsHelper {
}
}
ExternalIndexBulkModifyOperatorDescriptor treeIndexBulkLoadOp = new ExternalIndexBulkModifyOperatorDescriptor(
- spec, dataflowHelperFactory, deletedFiles, fieldPermutation, fillFactor, false, numElementsHint);
+ spec, dataflowHelperFactory, deletedFiles, fieldPermutation, fillFactor, false, numElementsHint, null);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, treeIndexBulkLoadOp,
secondaryPartitionConstraint);
return treeIndexBulkLoadOp;
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMIndexBulkLoadOperatorDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMIndexBulkLoadOperatorDescriptor.java
index 74590c7..ea84ca3 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMIndexBulkLoadOperatorDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMIndexBulkLoadOperatorDescriptor.java
@@ -25,6 +25,7 @@ import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
@@ -43,15 +44,19 @@ public class LSMIndexBulkLoadOperatorDescriptor extends TreeIndexBulkLoadOperato
protected final int datasetId;
+ protected final ITupleFilterFactory tupleFilterFactory;
+
public LSMIndexBulkLoadOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outRecDesc,
int[] fieldPermutation, float fillFactor, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex, IIndexDataflowHelperFactory indexHelperFactory,
- IIndexDataflowHelperFactory primaryIndexHelperFactory, BulkLoadUsage usage, int datasetId) {
+ IIndexDataflowHelperFactory primaryIndexHelperFactory, BulkLoadUsage usage, int datasetId,
+ ITupleFilterFactory tupleFilterFactory) {
super(spec, outRecDesc, fieldPermutation, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex,
indexHelperFactory);
this.primaryIndexHelperFactory = primaryIndexHelperFactory;
this.usage = usage;
this.datasetId = datasetId;
+ this.tupleFilterFactory = tupleFilterFactory;
}
@Override
@@ -59,6 +64,7 @@ public class LSMIndexBulkLoadOperatorDescriptor extends TreeIndexBulkLoadOperato
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new LSMIndexBulkLoadOperatorNodePushable(indexHelperFactory, primaryIndexHelperFactory, ctx, partition,
fieldPermutation, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex,
- recordDescProvider.getInputRecordDescriptor(this.getActivityId(), 0), usage, datasetId);
+ recordDescProvider.getInputRecordDescriptor(this.getActivityId(), 0), usage, datasetId,
+ tupleFilterFactory);
}
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMIndexBulkLoadOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMIndexBulkLoadOperatorNodePushable.java
index 4130490..46dfe65 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMIndexBulkLoadOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMIndexBulkLoadOperatorNodePushable.java
@@ -30,6 +30,7 @@ import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IndexBulkLoadOperatorNodePushable;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentId;
@@ -45,16 +46,15 @@ public class LSMIndexBulkLoadOperatorNodePushable extends IndexBulkLoadOperatorN
protected final IDatasetLifecycleManager datasetManager;
protected final int datasetId;
protected final int partition;
-
protected ILSMIndex primaryIndex;
public LSMIndexBulkLoadOperatorNodePushable(IIndexDataflowHelperFactory indexDataflowHelperFactory,
IIndexDataflowHelperFactory priamryIndexDataflowHelperFactory, IHyracksTaskContext ctx, int partition,
int[] fieldPermutation, float fillFactor, boolean verifyInput, long numElementsHint,
- boolean checkIfEmptyIndex, RecordDescriptor recDesc, BulkLoadUsage usage, int datasetId)
- throws HyracksDataException {
+ boolean checkIfEmptyIndex, RecordDescriptor recDesc, BulkLoadUsage usage, int datasetId,
+ ITupleFilterFactory tupleFilterFactory) throws HyracksDataException {
super(indexDataflowHelperFactory, ctx, partition, fieldPermutation, fillFactor, verifyInput, numElementsHint,
- checkIfEmptyIndex, recDesc);
+ checkIfEmptyIndex, recDesc, tupleFilterFactory);
if (priamryIndexDataflowHelperFactory != null) {
this.primaryIndexHelper =
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertOperatorNodePushable.java
index 2dc7f5e..a2e2a0b 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMSecondaryUpsertOperatorNodePushable.java
@@ -59,6 +59,7 @@ import org.apache.hyracks.storage.am.lsm.common.dataflow.LSMIndexInsertUpdateDel
*/
public class LSMSecondaryUpsertOperatorNodePushable extends LSMIndexInsertUpdateDeleteOperatorNodePushable {
+ private static final int NULL_MISSING_FIELD_INDEX = 0;
private final PermutingFrameTupleReference prevValueTuple = new PermutingFrameTupleReference();
private final int upsertIndicatorFieldIndex;
private final IBinaryBooleanInspector upsertIndicatorInspector;
@@ -104,9 +105,9 @@ public class LSMSecondaryUpsertOperatorNodePushable extends LSMIndexInsertUpdate
tuple.reset(accessor, i);
prevValueTuple.reset(accessor, i);
- boolean isNewValueMissing = isMissing(tuple, 0);
- boolean isOldValueMissing = isMissing(prevValueTuple, 0);
- if (isNewValueMissing && isOldValueMissing) {
+ boolean isNewValueNullOrMissing = isNullOrMissing(tuple);
+ boolean isOldValueNullOrMissing = isNullOrMissing(prevValueTuple);
+ if (isNewValueNullOrMissing && isOldValueNullOrMissing) {
// No op
continue;
}
@@ -117,12 +118,12 @@ public class LSMSecondaryUpsertOperatorNodePushable extends LSMIndexInsertUpdate
// which are always the same
continue;
}
- if (!isOldValueMissing) {
+ if (!isOldValueNullOrMissing) {
// We need to delete previous
abstractModCallback.setOp(Operation.DELETE);
lsmAccessor.forceDelete(prevValueTuple);
}
- if (isUpsert && !isNewValueMissing) {
+ if (isUpsert && !isNewValueNullOrMissing) {
// we need to insert the new value
abstractModCallback.setOp(Operation.INSERT);
lsmAccessor.forceInsert(tuple);
@@ -137,7 +138,8 @@ public class LSMSecondaryUpsertOperatorNodePushable extends LSMIndexInsertUpdate
FrameUtils.flushFrame(writeBuffer.getBuffer(), writer);
}
- private boolean isMissing(PermutingFrameTupleReference tuple, int fieldIdx) {
- return TypeTagUtil.isType(tuple, fieldIdx, ATypeTag.SERIALIZED_MISSING_TYPE_TAG);
+ private static boolean isNullOrMissing(PermutingFrameTupleReference tuple) {
+ return TypeTagUtil.isType(tuple, NULL_MISSING_FIELD_INDEX, ATypeTag.SERIALIZED_NULL_TYPE_TAG)
+ || TypeTagUtil.isType(tuple, NULL_MISSING_FIELD_INDEX, ATypeTag.SERIALIZED_MISSING_TYPE_TAG);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
index 095159b..c2a058c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
@@ -25,11 +25,15 @@ import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
+import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
+import org.apache.hyracks.storage.am.common.api.ITupleFilter;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.tuples.PermutingFrameTupleReference;
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.IIndexBulkLoader;
+import org.apache.hyracks.util.annotations.CriticalPath;
public class IndexBulkLoadOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
protected final IHyracksTaskContext ctx;
@@ -40,13 +44,17 @@ public class IndexBulkLoadOperatorNodePushable extends AbstractUnaryInputUnaryOu
protected final IIndexDataflowHelper indexHelper;
protected final RecordDescriptor recDesc;
protected final PermutingFrameTupleReference tuple = new PermutingFrameTupleReference();
+ protected final ITupleFilterFactory tupleFilterFactory;
protected FrameTupleAccessor accessor;
protected IIndex index;
protected IIndexBulkLoader bulkLoader;
+ protected ITupleFilter tupleFilter;
+ protected FrameTupleReference frameTuple;
public IndexBulkLoadOperatorNodePushable(IIndexDataflowHelperFactory indexDataflowHelperFactory,
IHyracksTaskContext ctx, int partition, int[] fieldPermutation, float fillFactor, boolean verifyInput,
- long numElementsHint, boolean checkIfEmptyIndex, RecordDescriptor recDesc) throws HyracksDataException {
+ long numElementsHint, boolean checkIfEmptyIndex, RecordDescriptor recDesc,
+ ITupleFilterFactory tupleFilterFactory) throws HyracksDataException {
this.ctx = ctx;
this.indexHelper = indexDataflowHelperFactory.create(ctx.getJobletContext().getServiceContext(), partition);
this.fillFactor = fillFactor;
@@ -54,8 +62,8 @@ public class IndexBulkLoadOperatorNodePushable extends AbstractUnaryInputUnaryOu
this.numElementsHint = numElementsHint;
this.checkIfEmptyIndex = checkIfEmptyIndex;
this.recDesc = recDesc;
+ this.tupleFilterFactory = tupleFilterFactory;
tuple.setFieldPermutation(fieldPermutation);
-
}
@Override
@@ -65,6 +73,10 @@ public class IndexBulkLoadOperatorNodePushable extends AbstractUnaryInputUnaryOu
index = indexHelper.getIndexInstance();
try {
writer.open();
+ if (tupleFilterFactory != null) {
+ tupleFilter = tupleFilterFactory.createTupleFilter(ctx);
+ frameTuple = new FrameTupleReference();
+ }
initializeBulkLoader();
} catch (Exception e) {
throw HyracksDataException.create(e);
@@ -75,8 +87,13 @@ public class IndexBulkLoadOperatorNodePushable extends AbstractUnaryInputUnaryOu
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
accessor.reset(buffer);
int tupleCount = accessor.getTupleCount();
-
for (int i = 0; i < tupleCount; i++) {
+ if (tupleFilter != null) {
+ frameTuple.reset(accessor, i);
+ if (!tupleFilter.accept(frameTuple)) {
+ continue;
+ }
+ }
tuple.reset(accessor, i);
bulkLoader.add(tuple);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
index fad344d..8346f62 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
@@ -26,6 +26,7 @@ import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
public class TreeIndexBulkLoadOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
@@ -37,10 +38,19 @@ public class TreeIndexBulkLoadOperatorDescriptor extends AbstractSingleActivityO
protected final long numElementsHint;
protected final boolean checkIfEmptyIndex;
protected final IIndexDataflowHelperFactory indexHelperFactory;
+ private final ITupleFilterFactory tupleFilterFactory;
public TreeIndexBulkLoadOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outRecDesc,
int[] fieldPermutation, float fillFactor, boolean verifyInput, long numElementsHint,
boolean checkIfEmptyIndex, IIndexDataflowHelperFactory indexHelperFactory) {
+ this(spec, outRecDesc, fieldPermutation, fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex,
+ indexHelperFactory, null);
+ }
+
+ public TreeIndexBulkLoadOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor outRecDesc,
+ int[] fieldPermutation, float fillFactor, boolean verifyInput, long numElementsHint,
+ boolean checkIfEmptyIndex, IIndexDataflowHelperFactory indexHelperFactory,
+ ITupleFilterFactory tupleFilterFactory) {
super(spec, 1, 1);
this.indexHelperFactory = indexHelperFactory;
this.fieldPermutation = fieldPermutation;
@@ -49,6 +59,7 @@ public class TreeIndexBulkLoadOperatorDescriptor extends AbstractSingleActivityO
this.numElementsHint = numElementsHint;
this.checkIfEmptyIndex = checkIfEmptyIndex;
this.outRecDescs[0] = outRecDesc;
+ this.tupleFilterFactory = tupleFilterFactory;
}
@Override
@@ -56,6 +67,6 @@ public class TreeIndexBulkLoadOperatorDescriptor extends AbstractSingleActivityO
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new IndexBulkLoadOperatorNodePushable(indexHelperFactory, ctx, partition, fieldPermutation, fillFactor,
verifyInput, numElementsHint, checkIfEmptyIndex,
- recordDescProvider.getInputRecordDescriptor(this.getActivityId(), 0));
+ recordDescProvider.getInputRecordDescriptor(this.getActivityId(), 0), tupleFilterFactory);
}
}
[asterixdb] 03/05: [NO ISSUE] Fix listing of conflicting procs on
sample start
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 01c8ed5d74890d78a0649da29b2e4d11ad3de5c1
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Apr 6 07:37:27 2019 -0400
[NO ISSUE] Fix listing of conflicting procs on sample start
Change-Id: I1d75156e13c1b03e58ebc52bd7f3546265cd5fd3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3328
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
index e0cff32..34d156c 100755
--- a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
+++ b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
@@ -127,7 +127,7 @@ if ps -ef | grep 'java.*org\.apache\.hyracks\.control\.[cn]c\.\([CN]CDriver\|ser
echo "aborting"
echo
echo "Re-run with -f to ignore, or run stop-sample-cluster.sh -f to forcibly terminate all running ${PRODUCT} processes:"
- cat /tmp/pids | sed 's/^ *[0-9]* \([0-9]*\).*org\.apache\.hyracks\.control\.[cn]c[^ ]*\.\([^ ]*\) .*/\1 - \2/'
+ cat /tmp/$$_pids | sed 's/^ *[0-9]* \([0-9]*\).*org\.apache\.hyracks\.control\.[cn]c[^ ]*\.\([^ ]*\) .*/\1 - \2/'
rm /tmp/$$_pids
exit 1
fi