You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by wi...@apache.org on 2013/02/19 13:51:59 UTC
[46/52] [partial] code contribution,
initial import of relevant modules of LMF-3.0.0-SNAPSHOT based on
revision 4bf944319368 of the default branch at https://code.google.com/p/lmf/
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/c32963d5/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/sqlmapper/LMFNativeEvaluationStrategy.java
----------------------------------------------------------------------
diff --git a/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/sqlmapper/LMFNativeEvaluationStrategy.java b/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/sqlmapper/LMFNativeEvaluationStrategy.java
new file mode 100644
index 0000000..fbe975e
--- /dev/null
+++ b/kiwi/kiwi-sparql/src/main/java/org/apache/marmotta/kiwi/sparql/sqlmapper/LMFNativeEvaluationStrategy.java
@@ -0,0 +1,477 @@
+/**
+ * Copyright (C) 2013 Salzburg Research.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package at.newmedialab.lmf.sparql.services.evaluation.sql;
+
+import at.newmedialab.sesame.commons.model.LiteralCommons;
+import info.aduna.iteration.CloseableIteration;
+import kiwi.core.api.config.ConfigurationService;
+import kiwi.core.api.persistence.PersistenceService;
+import kiwi.core.api.triplestore.SesameService;
+import kiwi.core.events.DBInitialisationEvent;
+import kiwi.core.model.rdf.KiWiNode;
+import kiwi.core.services.sail.LMFEvaluationStrategy;
+import org.apache.commons.validator.routines.UrlValidator;
+import org.jooq.Cursor;
+import org.jooq.Record;
+import org.jooq.SQLDialect;
+import org.jooq.Select;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.model.impl.LiteralImpl;
+import org.openrdf.model.impl.URIImpl;
+import org.openrdf.query.BindingSet;
+import org.openrdf.query.QueryEvaluationException;
+import org.openrdf.query.algebra.TupleExpr;
+import org.openrdf.query.algebra.ValueExpr;
+import org.openrdf.query.algebra.evaluation.ValueExprEvaluationException;
+import org.openrdf.query.impl.MapBindingSet;
+import org.slf4j.Logger;
+
+import javax.annotation.PostConstruct;
+import javax.enterprise.context.ApplicationScoped;
+import javax.enterprise.event.Observes;
+import javax.inject.Inject;
+import javax.persistence.EntityManager;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * An implementation of a Sesame evaluation strategy that translates SPARQL queries into native SQL queries
+ * on the LMF triple store for improved performance. This implementation is based on the jOOQ Java-to-SQL library
+ * to achieve a certain level of database independence.
+ *
+ * TODO: this class should somehow move to the lmf-sparql package
+ * <p/>
+ * Author: Sebastian Schaffert
+ */
+@ApplicationScoped
+public class LMFNativeEvaluationStrategy implements LMFEvaluationStrategy {
+
+ @Inject
+ private Logger log;
+
+ @Inject
+ private PersistenceService persistenceService;
+
+ @Inject
+ private ConfigurationService configurationService;
+
+
+ @Inject
+ private SesameService sesameService;
+
+
+ @PostConstruct
+ public void initialise() {
+ log.info("initialising LMF native query evaluation strategy (SPARQL -> SQL mapper)");
+
+ }
+
+
+ public void onDatabaseInitialisation(@Observes DBInitialisationEvent event) {
+ log.debug("Received DB-initialisation event, checking existance of SPARQL query indexes");
+
+ if(!"off".equals(configurationService.getStringConfiguration("database.mode")) && "native".equals(configurationService.getStringConfiguration("sparql.strategy"))) {
+ checkIndexes();
+ }
+ }
+
+
+ /**
+ * Depending on the database dialect, check for the existance of appropriate indexes to speed up
+ * typical SPARQL queries
+ */
+ private void checkIndexes() {
+ switch (getSQLDialect()) {
+ case POSTGRES:
+ log.info("SQL Dialect is PostgreSQL, creating advanced indexes");
+ createIndexesPostgres();
+ break;
+ }
+
+ }
+
+ private void createIndexesPostgres() {
+ Connection connection = persistenceService.getJDBCConnection();
+ try {
+ // index for language matches
+ String idx_sparql_lang = "create index idx_sparql_lang on kiwinode ( coalesce(locale,''));";
+ connection.createStatement().executeUpdate("drop index if exists idx_sparql_lang");
+ connection.createStatement().executeUpdate(idx_sparql_lang);
+
+ // index for string value matches
+ //String idx_sparql_content = "create index idx_sparql_content on kiwinode ( coalesce(content,uri,anonid,'') text_pattern_ops);";
+ String idx_sparql_content = "create index idx_sparql_content on kiwinode using hash( coalesce(content,uri,anonid,''));";
+ connection.createStatement().executeUpdate("drop index if exists idx_sparql_content");
+ connection.createStatement().executeUpdate(idx_sparql_content);
+
+ // index for combined URI/ID matches (typical join in triple patterns)
+ String idx_sparql_uri = "create unique index idx_sparql_uri on kiwinode (id, uri);";
+ connection.createStatement().executeUpdate("drop index if exists idx_sparql_uri");
+ connection.createStatement().executeUpdate(idx_sparql_uri);
+
+ if(!connection.getAutoCommit())
+ connection.commit();
+ } catch (SQLException e) {
+ log.error("Postgres: error while trying to create indexes!",e);
+ } finally {
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ log.error("could not close SQL connection",e);
+ }
+ }
+
+ }
+
+ /**
+ * Return the SQL Dialect used in this LMF instance. Returns null for unknown dialects.
+ *
+ * @return
+ */
+ private SQLDialect getSQLDialect() {
+ String db_type = configurationService.getStringConfiguration("database.type","h2");
+
+ SQLDialect dialect = null;
+ if(db_type.equals("h2")) {
+ dialect = SQLDialect.H2;
+ } else if(db_type.equals("postgres")) {
+ dialect = SQLDialect.POSTGRES;
+ } else if(db_type.equals("oracle")) {
+ dialect = SQLDialect.ORACLE;
+ } else if(db_type.equals("mysql")) {
+ dialect = SQLDialect.MYSQL;
+ }
+ return dialect;
+ }
+
+ /**
+ * Get the unique identifier for the evaluation strategy (e.g. "sesame", "hql", "native")
+ *
+ * @return
+ */
+ @Override
+ public String getIdentifier() {
+ return "native";
+ }
+
+ /**
+ * Return true if this evaluation strategy is available for use. Evaluation strategies can e.g. disable
+ * themselves in case the underlying database system or persistence layer is not supported.
+ *
+ * @return
+ */
+ @Override
+ public boolean isEnabled() {
+ String db_type = configurationService.getStringConfiguration("database.type","h2");
+
+ return "h2".equals(db_type) || "postgres".equals(db_type) || "mysql".equals(db_type) || "oracle".equals(db_type);
+ }
+
+ /**
+ * Evaluates the tuple expression against the supplied triple source with the
+ * specified set of variable bindings as input.
+ *
+ * @param expr The Tuple Expression to evaluate
+ * @param bindings The variables bindings to use for evaluating the expression, if
+ * applicable.
+ * @return A closeable iterator over the variable binding sets that match the
+ * tuple expression.
+ */
+ @Override
+ public CloseableIteration<BindingSet, QueryEvaluationException> evaluate(final TupleExpr expr, final BindingSet bindings) throws QueryEvaluationException {
+
+
+ String db_type = configurationService.getStringConfiguration("database.type","h2");
+
+ SQLDialect dialect = null;
+ if(db_type.equals("h2")) {
+ dialect = SQLDialect.H2;
+ } else if(db_type.equals("postgres")) {
+ dialect = SQLDialect.POSTGRES;
+ } else if(db_type.equals("oracle")) {
+ dialect = SQLDialect.ORACLE;
+ } else if(db_type.equals("mysql")) {
+ dialect = SQLDialect.MYSQL;
+ }
+
+ SparqlToSQLMapper.CastType castType = SparqlToSQLMapper.CastType.STRICT;
+ String castCfg = configurationService.getStringConfiguration("sparql.native.casttype","strict");
+ if("strict".equals(castCfg)) {
+ castType = SparqlToSQLMapper.CastType.STRICT;
+ } else if("loose".equals(castCfg)) {
+ castType = SparqlToSQLMapper.CastType.LOOSE;
+ } else if("none".equals(castCfg)) {
+ castType = SparqlToSQLMapper.CastType.NONE;
+ }
+
+ if(dialect != null) {
+
+ // rearrange the SPARQL abstract syntax tree so things are more suitable for SQL
+
+ // no unions as subselects in joins, we move the joins inside the union instead
+ new UnionOptimizer().optimize(expr);
+
+ final Connection connection = persistenceService.getJDBCConnection();
+
+ try {
+ connection.setAutoCommit(false);
+
+ final SparqlToSQLMapper mapper = new SparqlToSQLMapper(connection,dialect,expr, castType);
+
+ final Select query = mapper.getQuery();
+
+ if(log.isDebugEnabled() || configurationService.getBooleanConfiguration("sparql.native.logsql",true)) {
+ log.info("SQL query from SPARQL transformation:\n{}", query.getSQL());
+ }
+
+
+ final EntityManager em = persistenceService.getEntityManager();
+
+ return new CloseableIteration<BindingSet, QueryEvaluationException>() {
+
+ private Cursor cursor = query.fetchLazy(50);
+
+ private UrlValidator urlValidator = new UrlValidator(UrlValidator.ALLOW_ALL_SCHEMES | UrlValidator.ALLOW_LOCAL_URLS);
+
+ // in case we need to split one SQL query row into several SPARQL query rows, we do it by creating
+ // a temporary iterator
+ private Iterator<BindingSet> splitBindings = null;
+
+ @Override
+ public void close() throws QueryEvaluationException {
+ if(!cursor.isClosed()) {
+ cursor.close();
+ }
+ try {
+ if(!connection.getAutoCommit()) {
+ connection.commit();
+ }
+ connection.close();
+ } catch (SQLException e) {
+ log.error("SQL exception while closing JDBC connection",e);
+ }
+ em.close();
+ }
+
+ @Override
+ public boolean hasNext() throws QueryEvaluationException {
+ return (splitBindings !=null && splitBindings.hasNext()) || cursor.hasNext();
+ }
+
+ @Override
+ public BindingSet next() throws QueryEvaluationException {
+ if(splitBindings != null) {
+ return splitBindings.next();
+ } else {
+ Record record = cursor.fetchOne();
+
+ // check if the result contains multiple SPARQL bindings in a single row or
+ // only one SPARQL row; we can do this by checking whether the first result starts
+ // with _multi
+ if(record.getField(0).getName().startsWith("_multi")) {
+ log.info("transforming single-row SQL result into multi-row SPARQL result");
+ List<BindingSet> results = new ArrayList<BindingSet>();
+ for(int i=1; true; i++) {
+ MapBindingSet result = new MapBindingSet();
+ for(String var : mapper.getProjectedVariables()) {
+ if(var.startsWith("_multi") && var.endsWith("_"+i)) {
+ Long nodeId = record.getValue(var, Long.class);
+
+ if(nodeId != null) {
+ Value value = em.find(KiWiNode.class, nodeId);
+
+ result.addBinding(var.substring(var.indexOf('_',1)+1,var.lastIndexOf('_')),value);
+ }
+ }
+ }
+ for(Map.Entry<String,Class> ext : mapper.getExtensionVariables().entrySet()) {
+ String var = ext.getKey();
+ if(var.startsWith("_multi") && var.endsWith("_"+i)) {
+ Object val = record.getValue(ext.getKey(),ext.getValue());
+
+ // this is truly a hack: we check whether the string is a URI, and if yes create a URI resource...
+ // it would be better to carry over this information from the value constants
+ if(urlValidator.isValid(val.toString())) {
+ URI value = new URIImpl(val.toString());
+ result.addBinding(var.substring(var.indexOf('_',1)+1,var.lastIndexOf('_')),value);
+ } else {
+ String type = LiteralCommons.getXSDType(ext.getValue());
+
+ // we only create an in-memory representation of the value, the LMF methods
+ // would automatically persist it, so we create a Sesame value
+ Value value = new LiteralImpl(val.toString(),sesameService.getValueFactory().createURI(type));
+ result.addBinding(var.substring(var.indexOf('_',1)+1,var.lastIndexOf('_')),value);
+ }
+ }
+ }
+ if(result.size() == 0) {
+ break;
+ } else {
+ results.add(result);
+ }
+ }
+ em.clear();
+ splitBindings = results.iterator();
+ return splitBindings.next();
+ } else {
+
+ MapBindingSet result = new MapBindingSet();
+ for(String var : mapper.getProjectedVariables()) {
+ Long nodeId = record.getValue(var, Long.class);
+
+ if(nodeId != null) {
+ Value value = em.find(KiWiNode.class, nodeId);
+ result.addBinding(var,value);
+ }
+ }
+ for(Map.Entry<String,Class> ext : mapper.getExtensionVariables().entrySet()) {
+ Object val = record.getValue(ext.getKey(),ext.getValue());
+
+ // this is truly a hack: we check whether the string is a URI, and if yes create a URI resource...
+ // it would be better to carry over this information from the value constants
+ if(urlValidator.isValid(val.toString())) {
+ URI value = new URIImpl(val.toString());
+ result.addBinding(ext.getKey(),value);
+ } else {
+ String type = LiteralCommons.getXSDType(ext.getValue());
+
+ // we only create an in-memory representation of the value, the LMF methods
+ // would automatically persist it, so we create a Sesame value
+ Value value = new LiteralImpl(val.toString(),sesameService.getValueFactory().createURI(type));
+ result.addBinding(ext.getKey(),value);
+ }
+ }
+
+ em.clear();
+
+ return result;
+ }
+ }
+ }
+
+ @Override
+ public void remove() throws QueryEvaluationException {
+ throw new UnsupportedOperationException("removing not supported");
+ }
+ };
+
+ } catch(Exception ex) {
+ log.error("exception while translating SPARQL query",ex);
+ log.error("abstract query tree was: {}",expr);
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ log.error("error while trying to close JDBC connection",e);
+ }
+ }
+
+ throw new UnsupportedOperationException("not yet implemented");
+ }
+
+ throw new UnsupportedOperationException("the database "+db_type+" is not yet supported for SPARQL->SQL mapping");
+ }
+
+ /**
+ * Gets the value of this expression.
+ *
+ * @param bindings The variables bindings to use for evaluating the expression, if
+ * applicable.
+ * @return The Value that this expression evaluates to, or <tt>null</tt> if
+ * the expression could not be evaluated.
+ */
+ @Override
+ public Value evaluate(ValueExpr expr, BindingSet bindings) throws ValueExprEvaluationException, QueryEvaluationException {
+ String db_type = configurationService.getStringConfiguration("database.type","h2");
+ /*
+ SQLDialect dialect = null;
+ if(db_type.equals("h2")) {
+ dialect = SQLDialect.H2;
+ } else if(db_type.equals("postgres")) {
+ dialect = SQLDialect.POSTGRES;
+ } else if(db_type.equals("oracle")) {
+ dialect = SQLDialect.ORACLE;
+ } else if(db_type.equals("mysql")) {
+ dialect = SQLDialect.MYSQL;
+ }
+
+ SparqlToSQLMapper.CastType castType = SparqlToSQLMapper.CastType.STRICT;
+ String castCfg = configurationService.getStringConfiguration("sparql.native.casttype","strict");
+ if("strict".equals(castCfg)) {
+ castType = SparqlToSQLMapper.CastType.STRICT;
+ } else if("loose".equals(castCfg)) {
+ castType = SparqlToSQLMapper.CastType.LOOSE;
+ } else if("none".equals(castCfg)) {
+ castType = SparqlToSQLMapper.CastType.NONE;
+ }
+
+ if(dialect != null) {
+ final Connection connection = persistenceService.getJDBCConnection();
+
+ try {
+ final SparqlToSQLMapper mapper = new SparqlToSQLMapper(connection,dialect,expr,castType);
+
+ final Select query = mapper.getQuery();
+
+ log.info("SQL query would be: {}", query.getSQL());
+
+
+ Object value = query.fetchOne(0);
+
+
+ connection.commit();
+ connection.close();
+
+ throw new UnsupportedOperationException("not yet implemented");
+ } catch (SQLException e) {
+ log.error("error while evaluating SQL query",e);
+ } finally {
+ try {
+ connection.close();
+ } catch (SQLException e) {
+ log.error("error while trying to close JDBC connection",e);
+ }
+ }
+
+ }
+*/
+ throw new UnsupportedOperationException("the database "+db_type+" is not yet supported for SPARQL->SQL mapping");
+ }
+
+ /**
+ * Evaluates the boolean expression on the supplied TripleSource object.
+ *
+ * @param bindings The variables bindings to use for evaluating the expression, if
+ * applicable.
+ * @return The result of the evaluation.
+ * @throws org.openrdf.query.algebra.evaluation.ValueExprEvaluationException
+ * If the value expression could not be evaluated, for example when
+ * comparing two incompatible operands. When thrown, the result of
+ * the boolean expression is neither <tt>true</tt> nor
+ * <tt>false</tt>, but unknown.
+ */
+ @Override
+ public boolean isTrue(ValueExpr expr, BindingSet bindings) throws ValueExprEvaluationException, QueryEvaluationException {
+ return false; //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+
+
+}