You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by tk...@apache.org on 2016/04/20 09:45:57 UTC
[1/3] marmotta git commit: prevents filter optimization if function
is not implemented natively
Repository: marmotta
Updated Branches:
refs/heads/develop ff559ea4f -> c8f067f0f
prevents filter optimization if function is not implemented natively
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/b79a1621
Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/b79a1621
Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/b79a1621
Branch: refs/heads/develop
Commit: b79a16213a403ab2bd82b3da714a6a681d046083
Parents: c008f95
Author: Thomas Kurz <th...@salzburgresearch.at>
Authored: Mon Nov 23 13:13:15 2015 +0100
Committer: Thomas Kurz <th...@salzburgresearch.at>
Committed: Mon Nov 23 13:13:15 2015 +0100
----------------------------------------------------------------------
.../sparql/optimizer/NativeFilterOptimizer.java | 198 +++++++++++++++++++
.../sparql/sail/KiWiSparqlSailConnection.java | 5 +-
2 files changed, 201 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/marmotta/blob/b79a1621/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/optimizer/NativeFilterOptimizer.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/optimizer/NativeFilterOptimizer.java b/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/optimizer/NativeFilterOptimizer.java
new file mode 100644
index 0000000..5cde939
--- /dev/null
+++ b/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/optimizer/NativeFilterOptimizer.java
@@ -0,0 +1,198 @@
+package org.apache.marmotta.kiwi.sparql.optimizer;
+
+import org.apache.marmotta.kiwi.sparql.function.NativeFunctionRegistry;
+import org.openrdf.model.URI;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.Dataset;
+import org.openrdf.query.algebra.*;
+import org.openrdf.query.algebra.evaluation.QueryOptimizer;
+import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
+import org.openrdf.query.algebra.helpers.VarNameCollector;
+
+import java.util.Set;
+
+/**
+ * @author Thomas Kurz (tkurz@apache.org)
+ * @since 23.11.15.
+ */
+public class NativeFilterOptimizer implements QueryOptimizer {
+
+ @Override
+ public void optimize(TupleExpr tupleExpr, Dataset dataset, BindingSet bindings) {
+ FilterFinder f = new FilterFinder(tupleExpr);
+ tupleExpr.visit(f);
+ }
+
+ /*--------------------------*
+ * Inner class FilterFinder *
+ *--------------------------*/
+
+ protected static class FilterFinder extends QueryModelVisitorBase<RuntimeException> {
+
+ protected final TupleExpr tupleExpr;
+
+ public FilterFinder(TupleExpr tupleExpr) {
+ this.tupleExpr = tupleExpr;
+ }
+
+ @Override
+ public void meet(Filter filter) {
+ super.meet(filter);
+ if(filter.getCondition() instanceof FunctionCall) {
+ String uri = ((FunctionCall) filter.getCondition()).getURI();
+ if(NativeFunctionRegistry.getInstance().get(uri) != null) {
+ FilterRelocator.relocate(filter);
+ }
+ } else {
+ FilterRelocator.relocate(filter);
+ }
+ }
+ }
+
+
+ /*-----------------------------*
+ * Inner class FilterRelocator *
+ *-----------------------------*/
+
+ protected static class FilterRelocator extends QueryModelVisitorBase<RuntimeException> {
+
+ public static void relocate(Filter filter) {
+ filter.visit(new FilterRelocator(filter));
+ }
+
+ protected final Filter filter;
+
+ protected final Set<String> filterVars;
+
+ public FilterRelocator(Filter filter) {
+ this.filter = filter;
+ filterVars = VarNameCollector.process(filter.getCondition());
+ }
+
+ @Override
+ protected void meetNode(QueryModelNode node) {
+ // By default, do not traverse
+ assert node instanceof TupleExpr;
+ relocate(filter, (TupleExpr) node);
+ }
+
+ @Override
+ public void meet(Join join) {
+ if (join.getLeftArg().getBindingNames().containsAll(filterVars)) {
+ // All required vars are bound by the left expr
+ join.getLeftArg().visit(this);
+ }
+ else if (join.getRightArg().getBindingNames().containsAll(filterVars)) {
+ // All required vars are bound by the right expr
+ join.getRightArg().visit(this);
+ }
+ else {
+ relocate(filter, join);
+ }
+ }
+
+ @Override
+ public void meet(LeftJoin leftJoin) {
+ if (leftJoin.getLeftArg().getBindingNames().containsAll(filterVars)) {
+ leftJoin.getLeftArg().visit(this);
+ }
+ else {
+ relocate(filter, leftJoin);
+ }
+ }
+
+ @Override
+ public void meet(Union union) {
+ Filter clone = new Filter();
+ clone.setCondition(filter.getCondition().clone());
+
+ relocate(filter, union.getLeftArg());
+ relocate(clone, union.getRightArg());
+
+ FilterRelocator.relocate(filter);
+ FilterRelocator.relocate(clone);
+ }
+
+ @Override
+ public void meet(Difference node) {
+ Filter clone = new Filter();
+ clone.setCondition(filter.getCondition().clone());
+
+ relocate(filter, node.getLeftArg());
+ relocate(clone, node.getRightArg());
+
+ FilterRelocator.relocate(filter);
+ FilterRelocator.relocate(clone);
+ }
+
+ @Override
+ public void meet(Intersection node) {
+ Filter clone = new Filter();
+ clone.setCondition(filter.getCondition().clone());
+
+ relocate(filter, node.getLeftArg());
+ relocate(clone, node.getRightArg());
+
+ FilterRelocator.relocate(filter);
+ FilterRelocator.relocate(clone);
+ }
+
+ @Override
+ public void meet(Extension node) {
+ if (node.getArg().getBindingNames().containsAll(filterVars)) {
+ node.getArg().visit(this);
+ }
+ else {
+ relocate(filter, node);
+ }
+ }
+
+ @Override
+ public void meet(EmptySet node) {
+ if (filter.getParentNode() != null) {
+ // Remove filter from its original location
+ filter.replaceWith(filter.getArg());
+ }
+ }
+
+ @Override
+ public void meet(Filter filter) {
+ // Filters are commutative
+ filter.getArg().visit(this);
+ }
+
+ @Override
+ public void meet(Distinct node) {
+ node.getArg().visit(this);
+ }
+
+ @Override
+ public void meet(Order node) {
+ node.getArg().visit(this);
+ }
+
+ @Override
+ public void meet(QueryRoot node) {
+ node.getArg().visit(this);
+ }
+
+ @Override
+ public void meet(Reduced node) {
+ node.getArg().visit(this);
+ }
+
+ protected void relocate(Filter filter, TupleExpr newFilterArg) {
+ if (filter.getArg() != newFilterArg) {
+ if (filter.getParentNode() != null) {
+ // Remove filter from its original location
+ filter.replaceWith(filter.getArg());
+ }
+
+ // Insert filter at the new location
+ newFilterArg.replaceWith(filter);
+ filter.setArg(newFilterArg);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/marmotta/blob/b79a1621/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/sail/KiWiSparqlSailConnection.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/sail/KiWiSparqlSailConnection.java b/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/sail/KiWiSparqlSailConnection.java
index 09333e7..3c77ead 100644
--- a/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/sail/KiWiSparqlSailConnection.java
+++ b/libraries/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/sail/KiWiSparqlSailConnection.java
@@ -25,6 +25,7 @@ import org.apache.marmotta.kiwi.sparql.evaluation.KiWiEvaluationStrategy;
import org.apache.marmotta.kiwi.sparql.evaluation.KiWiTripleSource;
import org.apache.marmotta.kiwi.sparql.optimizer.DifferenceOptimizer;
import org.apache.marmotta.kiwi.sparql.optimizer.DistinctLimitOptimizer;
+import org.apache.marmotta.kiwi.sparql.optimizer.NativeFilterOptimizer;
import org.openrdf.query.BindingSet;
import org.openrdf.query.Dataset;
import org.openrdf.query.QueryEvaluationException;
@@ -81,11 +82,11 @@ public class KiWiSparqlSailConnection extends NotifyingSailConnectionWrapper {
//new DisjunctiveConstraintOptimizer().optimize(tupleExpr, dataset, bindings);
//new SameTermFilterOptimizer().optimize(tupleExpr, dataset, bindings);
-
new QueryModelNormalizer().optimize(tupleExpr, dataset, bindings);
new QueryJoinOptimizer(new KiWiEvaluationStatistics()).optimize(tupleExpr, dataset, bindings);
new IterativeEvaluationOptimizer().optimize(tupleExpr, dataset, bindings);
- new FilterOptimizer().optimize(tupleExpr, dataset, bindings);
+
+ new NativeFilterOptimizer().optimize(tupleExpr, dataset, bindings);
//new OrderLimitOptimizer().optimize(tupleExpr, dataset, bindings);
new DistinctLimitOptimizer().optimize(tupleExpr, dataset, bindings);
[2/3] marmotta git commit: Merge branch 'develop' of
https://git-wip-us.apache.org/repos/asf/marmotta into develop
Posted by tk...@apache.org.
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/marmotta into develop
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/28e6dba3
Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/28e6dba3
Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/28e6dba3
Branch: refs/heads/develop
Commit: 28e6dba31cc45614ce8b2ac2602b3f986a527de9
Parents: b79a162 503cbc8
Author: Thomas Kurz <th...@salzburgresearch.at>
Authored: Tue Mar 8 15:12:07 2016 +0100
Committer: Thomas Kurz <th...@salzburgresearch.at>
Committed: Tue Mar 8 15:12:07 2016 +0100
----------------------------------------------------------------------
README.md | 109 +
README.txt | 119 -
launchers/marmotta-webapp/README.md | 1 +
launchers/marmotta-webapp/pom.xml | 22 +
.../kiwi/reasoner/parser/KWRLProgramParser.jj | 26 +-
.../test/parser/KWRLRuleParserTest.java | 41 +-
.../persistence/KWRLProgramPersistenceTest.java | 2 -
.../builder/eval/ValueExpressionEvaluator.java | 6 +-
.../kiwi/sparql/test/KiWiSparqlTest.java | 56 +-
.../marmotta/kiwi/sail/KiWiValueFactory.java | 28 +-
libraries/ostrich/Dockerfile | 77 +
libraries/ostrich/README.md | 105 +
libraries/ostrich/backend/.dockerignore | 1 +
libraries/ostrich/backend/CMakeLists.txt | 42 +
libraries/ostrich/backend/client/CMakeLists.txt | 9 +
libraries/ostrich/backend/client/client.cc | 379 +
.../ostrich/backend/cmake/FindGFlags.cmake | 48 +
libraries/ostrich/backend/cmake/FindGLog.cmake | 18 +
libraries/ostrich/backend/cmake/FindGRPC.cmake | 67 +
.../ostrich/backend/cmake/FindLevelDB.cmake | 18 +
.../ostrich/backend/cmake/FindRAPTOR.cmake | 103 +
.../ostrich/backend/cmake/FindRasqal.cmake | 99 +
.../ostrich/backend/cmake/FindTcmalloc.cmake | 39 +
libraries/ostrich/backend/model/CMakeLists.txt | 6 +
libraries/ostrich/backend/model/model.proto | 77 +
libraries/ostrich/backend/model/rdf_model.cc | 348 +
libraries/ostrich/backend/model/rdf_model.h | 709 +
.../ostrich/backend/model/rdf_operators.cc | 65 +
libraries/ostrich/backend/model/rdf_operators.h | 240 +
libraries/ostrich/backend/parser/CMakeLists.txt | 6 +
libraries/ostrich/backend/parser/rdf_parser.cc | 157 +
libraries/ostrich/backend/parser/rdf_parser.h | 94 +
.../ostrich/backend/persistence/CMakeLists.txt | 21 +
.../backend/persistence/leveldb_persistence.cc | 729 +
.../backend/persistence/leveldb_persistence.h | 185 +
.../backend/persistence/leveldb_server.cc | 73 +
.../backend/persistence/leveldb_service.cc | 307 +
.../backend/persistence/leveldb_service.h | 128 +
.../backend/persistence/leveldb_sparql.cc | 85 +
.../backend/persistence/leveldb_sparql.h | 52 +
.../backend/persistence/marmotta_updatedb.cc | 215 +
.../ostrich/backend/serializer/CMakeLists.txt | 10 +
.../ostrich/backend/serializer/serializer.cc | 49 +
.../ostrich/backend/serializer/serializer.h | 54 +
.../backend/serializer/serializer_base.cc | 76 +
.../backend/serializer/serializer_base.h | 104 +
.../backend/serializer/serializer_proto.cc | 56 +
.../backend/serializer/serializer_proto.h | 50 +
.../backend/serializer/serializer_raptor.cc | 175 +
.../backend/serializer/serializer_raptor.h | 55 +
.../ostrich/backend/service/CMakeLists.txt | 9 +
libraries/ostrich/backend/service/sail.proto | 102 +
libraries/ostrich/backend/service/sparql.proto | 53 +
.../ostrich/backend/sharding/CMakeLists.txt | 11 +
libraries/ostrich/backend/sharding/server.cc | 66 +
libraries/ostrich/backend/sharding/sharding.cc | 335 +
libraries/ostrich/backend/sharding/sharding.h | 174 +
libraries/ostrich/backend/sparql/CMakeLists.txt | 8 +
.../ostrich/backend/sparql/rasqal_adapter.cc | 382 +
.../ostrich/backend/sparql/rasqal_adapter.h | 124 +
.../ostrich/backend/sparql/rasqal_model.cc | 205 +
libraries/ostrich/backend/sparql/rasqal_model.h | 73 +
libraries/ostrich/backend/test/CMakeLists.txt | 20 +
.../ostrich/backend/test/PersistenceTest.cc | 268 +
libraries/ostrich/backend/test/SparqlTest.cc | 291 +
libraries/ostrich/backend/test/StatementTest.cc | 135 +
.../ostrich/backend/test/gmock-gtest-all.cc | 12243 ++++++++++
libraries/ostrich/backend/test/gmock/gmock.h | 14978 ++++++++++++
libraries/ostrich/backend/test/gtest/gtest.h | 21197 +++++++++++++++++
libraries/ostrich/backend/test/main.cc | 11 +
libraries/ostrich/backend/util/CMakeLists.txt | 6 +
libraries/ostrich/backend/util/iterator.h | 315 +
libraries/ostrich/backend/util/murmur3.cc | 313 +
libraries/ostrich/backend/util/murmur3.h | 18 +
libraries/ostrich/backend/util/raptor_util.cc | 179 +
libraries/ostrich/backend/util/raptor_util.h | 74 +
libraries/ostrich/backend/util/split.cc | 40 +
libraries/ostrich/backend/util/split.h | 38 +
libraries/ostrich/backend/util/time_logger.cc | 17 +
libraries/ostrich/backend/util/time_logger.h | 32 +
libraries/ostrich/backend/util/unique.h | 23 +
libraries/ostrich/client/pom.xml | 234 +
.../ostrich/sail/ClosableResponseStream.java | 163 +
.../marmotta/ostrich/sail/OstrichSail.java | 87 +
.../ostrich/sail/OstrichSailConnection.java | 618 +
.../ostrich/sail/OstrichValueFactory.java | 260 +
.../ostrich/sail/test/CMarmottaSailTest.java | 74 +
.../ostrich/sail/test/TestSailConnection.java | 80 +
libraries/ostrich/model/pom.xml | 219 +
.../marmotta/ostrich/model/ProtoBNode.java | 79 +
.../ostrich/model/ProtoDatatypeLiteral.java | 106 +
.../ostrich/model/ProtoLiteralBase.java | 187 +
.../marmotta/ostrich/model/ProtoNamespace.java | 86 +
.../marmotta/ostrich/model/ProtoStatement.java | 221 +
.../ostrich/model/ProtoStringLiteral.java | 101 +
.../apache/marmotta/ostrich/model/ProtoURI.java | 113 +
.../ostrich/model/test/StatementTest.java | 68 +
.../marmotta/ostrich/model/test/URITest.java | 57 +
.../model/src/test/resources/logback.xml | 28 +
libraries/ostrich/pom.xml | 60 +
libraries/pom.xml | 9 +
loader/marmotta-loader-ostrich/pom.xml | 83 +
.../loader/ostrich/OstrichLoaderBackend.java | 100 +
.../loader/ostrich/OstrichLoaderHandler.java | 156 +
...org.apache.marmotta.loader.api.LoaderBackend | 18 +
.../src/main/resources/logback.xml | 32 +
loader/pom.xml | 10 +
.../backends/marmotta-backend-ostrich/pom.xml | 176 +
.../backend/ostrich/OstrichProvider.java | 97 +
.../backend/ostrich/OstrichSailRepository.java | 168 +
.../src/main/resources/META-INF/beans.xml | 28 +
.../main/resources/config-defaults.properties | 25 +
.../resources/config-descriptions.properties | 31 +
.../src/main/resources/kiwi-module.properties | 38 +
.../src/main/resources/web/admin/about.html | 36 +
.../main/resources/web/admin/configuration.html | 52 +
platform/backends/pom.xml | 6 +
.../triplestore/ContextServiceImpl.java | 82 +-
.../test/triplestore/ContextServiceTest.java | 78 +
119 files changed, 61197 insertions(+), 150 deletions(-)
----------------------------------------------------------------------
[3/3] marmotta git commit: Merge branch 'develop' of
https://git-wip-us.apache.org/repos/asf/marmotta into develop
Posted by tk...@apache.org.
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/marmotta into develop
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/c8f067f0
Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/c8f067f0
Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/c8f067f0
Branch: refs/heads/develop
Commit: c8f067f0f0d6735015f26bcc386b3edc1c46cd6d
Parents: 28e6dba ff559ea
Author: Thomas Kurz <th...@salzburgresearch.at>
Authored: Wed Apr 20 09:42:56 2016 +0200
Committer: Thomas Kurz <th...@salzburgresearch.at>
Committed: Wed Apr 20 09:42:56 2016 +0200
----------------------------------------------------------------------
.../services/test/ng/BaseLDCacheTest.java | 7 +---
.../test/rdf/TestLinkedDataProvider.java | 1 +
parent/pom.xml | 2 +-
.../src/main/resources/web/admin/about.html | 41 ++++++--------------
.../main/resources/web/admin/configuration.html | 7 +---
.../src/main/resources/web/admin/console.html | 2 +-
.../src/main/resources/web/admin/database.html | 2 +-
7 files changed, 20 insertions(+), 42 deletions(-)
----------------------------------------------------------------------