You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@rya.apache.org by mi...@apache.org on 2015/12/07 13:05:10 UTC
[40/51] [partial] incubator-rya git commit: Cannot delete temp branch,
doc'd it.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/accumulo/mr/fileinput/RyaBatchWriterInputTool.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/accumulo/mr/fileinput/RyaBatchWriterInputTool.java b/extras/indexing/src/main/java/mvm/rya/accumulo/mr/fileinput/RyaBatchWriterInputTool.java
deleted file mode 100644
index fb80804..0000000
--- a/extras/indexing/src/main/java/mvm/rya/accumulo/mr/fileinput/RyaBatchWriterInputTool.java
+++ /dev/null
@@ -1,243 +0,0 @@
-package mvm.rya.accumulo.mr.fileinput;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.io.IOException;
-import java.io.StringReader;
-
-import mvm.rya.accumulo.mr.RyaOutputFormat;
-import mvm.rya.accumulo.mr.StatementWritable;
-import mvm.rya.accumulo.mr.utils.MRUtils;
-import mvm.rya.indexing.accumulo.ConfigUtils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.log4j.Logger;
-import org.openrdf.model.Resource;
-import org.openrdf.model.Statement;
-import org.openrdf.model.URI;
-import org.openrdf.model.Value;
-import org.openrdf.model.ValueFactory;
-import org.openrdf.model.impl.ValueFactoryImpl;
-import org.openrdf.rio.ParserConfig;
-import org.openrdf.rio.RDFFormat;
-import org.openrdf.rio.RDFHandlerException;
-import org.openrdf.rio.RDFParseException;
-import org.openrdf.rio.RDFParser;
-import org.openrdf.rio.Rio;
-import org.openrdf.rio.helpers.RDFHandlerBase;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Take large ntrips files and use MapReduce to ingest into other indexing
- */
-public class RyaBatchWriterInputTool extends Configured implements Tool {
- private static final Logger logger = Logger.getLogger(RyaBatchWriterInputTool.class);
-
- @Override
- public int run(final String[] args) throws Exception {
- String userName = null;
- String pwd = null;
- String instance = null;
- String zk = null;
- String format = null;
-
- final Configuration conf = getConf();
- // conf
- zk = conf.get(MRUtils.AC_ZK_PROP, zk);
- instance = conf.get(MRUtils.AC_INSTANCE_PROP, instance);
- userName = conf.get(MRUtils.AC_USERNAME_PROP, userName);
- pwd = conf.get(MRUtils.AC_PWD_PROP, pwd);
- format = conf.get(MRUtils.FORMAT_PROP, RDFFormat.NTRIPLES.getName());
-
- String auths = conf.get(MRUtils.AC_CV_PROP, "");
-
- conf.set(MRUtils.FORMAT_PROP, format);
- Preconditions.checkNotNull(zk, MRUtils.AC_ZK_PROP + " not set");
- Preconditions.checkNotNull(instance, MRUtils.AC_INSTANCE_PROP + " not set");
- Preconditions.checkNotNull(userName, MRUtils.AC_USERNAME_PROP + " not set");
- Preconditions.checkNotNull(pwd, MRUtils.AC_PWD_PROP + " not set");
-
- // map the config values to free text configure values
- conf.set(ConfigUtils.CLOUDBASE_ZOOKEEPERS, zk);
- conf.set(ConfigUtils.CLOUDBASE_INSTANCE, instance);
- conf.set(ConfigUtils.CLOUDBASE_USER, userName);
- conf.set(ConfigUtils.CLOUDBASE_PASSWORD, pwd);
- conf.set(ConfigUtils.CLOUDBASE_AUTHS, auths);
-
- final String inputDir = args[0];
-
- String tablePrefix = conf.get(MRUtils.TABLE_PREFIX_PROPERTY, null);
- Preconditions.checkNotNull(tablePrefix, MRUtils.TABLE_PREFIX_PROPERTY + " not set");
-
- String docTextTable = tablePrefix + "text";
- conf.set(ConfigUtils.FREE_TEXT_DOC_TABLENAME, docTextTable);
-
- String docTermTable = tablePrefix + "terms";
- conf.set(ConfigUtils.FREE_TEXT_TERM_TABLENAME, docTermTable);
-
- String geoTable = tablePrefix + "geo";
- conf.set(ConfigUtils.GEO_TABLENAME, geoTable);
-
- logger.info("Loading data into tables[rya, freetext, geo]");
- logger.info("Loading data into tables[" + docTermTable + " " + docTextTable + " " + geoTable + "]");
-
- Job job = new Job(new Configuration(conf), "Batch Writer load data into Rya Core and Indexing Tables");
- job.setJarByClass(this.getClass());
-
- // setting long job
- Configuration jobConf = job.getConfiguration();
- jobConf.setBoolean("mapred.map.tasks.speculative.execution", false);
-
- jobConf.setInt("mapred.task.timeout", 1000 * 60 * 60 * 24); // timeout after 1 day
-
- job.setInputFormatClass(TextInputFormat.class);
-
- job.setMapperClass(ParseNtripsMapper.class);
-
- job.setNumReduceTasks(0);
-
- // Use Rya Output Format
- job.setOutputFormatClass(RyaOutputFormat.class);
- job.setOutputKeyClass(NullWritable.class);
- job.setOutputValueClass(StatementWritable.class);
- job.setMapOutputKeyClass(NullWritable.class);
- job.setMapOutputValueClass(StatementWritable.class);
-
- TextInputFormat.setInputPaths(job, new Path(inputDir));
-
- job.waitForCompletion(true);
-
- return 0;
- }
-
- public static void main(String[] args) throws Exception {
- ToolRunner.run(new Configuration(), new RyaBatchWriterInputTool(), args);
- }
-
- public static class ParseNtripsMapper extends Mapper<LongWritable, Text, Writable, Statement> {
- private static final Logger logger = Logger.getLogger(ParseNtripsMapper.class);
-
- private RDFParser parser;
- private RDFFormat rdfFormat;
-
- @Override
- protected void setup(final Context context) throws IOException, InterruptedException {
- super.setup(context);
- Configuration conf = context.getConfiguration();
-
- final ValueFactory vf = new ValueFactoryImpl();
-
- String rdfFormatName = conf.get(MRUtils.FORMAT_PROP);
- checkNotNull(rdfFormatName, "Rdf format cannot be null");
- rdfFormat = RDFFormat.valueOf(rdfFormatName);
-
- String namedGraphString = conf.get(MRUtils.NAMED_GRAPH_PROP);
- checkNotNull(namedGraphString, MRUtils.NAMED_GRAPH_PROP + " cannot be null");
-
- final Resource namedGraph = vf.createURI(namedGraphString);
-
- parser = Rio.createParser(rdfFormat);
- parser.setParserConfig(new ParserConfig(true, true, true, RDFParser.DatatypeHandling.VERIFY));
- parser.setRDFHandler(new RDFHandlerBase() {
- @Override
- public void handleStatement(Statement statement) throws RDFHandlerException {
- Statement output;
- if (rdfFormat.equals(RDFFormat.NTRIPLES)) {
- output = new ConextStatementWrapper(statement, namedGraph);
- } else {
- output = statement;
- }
- try {
- context.write(NullWritable.get(), new StatementWritable(output));
- } catch (IOException e) {
- logger.error("Error writing statement", e);
- throw new RDFHandlerException(e);
- } catch (InterruptedException e) {
- logger.error("Error writing statement", e);
- throw new RDFHandlerException(e);
- }
- }
-
- });
- }
-
- @Override
- public void map(LongWritable key, Text value, Context output) throws IOException, InterruptedException {
- String rdf = value.toString();
- try {
- parser.parse(new StringReader(rdf), "");
- } catch (RDFParseException e) {
- logger.error("Line[" + rdf + "] cannot be formatted with format[" + rdfFormat + "]. Exception[" + e.getMessage()
- + "]", e);
- } catch (Exception e) {
- logger.error("error during map", e);
- throw new IOException("Exception occurred parsing triple[" + rdf + "]", e);
- }
- }
- }
-
- @SuppressWarnings("serial")
- private static class ConextStatementWrapper implements Statement {
- private Statement statementWithoutConext;
- private Resource context;
-
- public ConextStatementWrapper(Statement statementWithoutConext, Resource context) {
- this.statementWithoutConext = statementWithoutConext;
- this.context = context;
- }
-
- @Override
- public Resource getSubject() {
- return statementWithoutConext.getSubject();
- }
-
- @Override
- public URI getPredicate() {
- return statementWithoutConext.getPredicate();
- }
-
- @Override
- public Value getObject() {
- return statementWithoutConext.getObject();
- }
-
- @Override
- public Resource getContext() {
- return context;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/accumulo/precompQuery/AccumuloPrecompQueryIndexer.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/accumulo/precompQuery/AccumuloPrecompQueryIndexer.java b/extras/indexing/src/main/java/mvm/rya/accumulo/precompQuery/AccumuloPrecompQueryIndexer.java
deleted file mode 100644
index 86cb73e..0000000
--- a/extras/indexing/src/main/java/mvm/rya/accumulo/precompQuery/AccumuloPrecompQueryIndexer.java
+++ /dev/null
@@ -1,326 +0,0 @@
-package mvm.rya.accumulo.precompQuery;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import info.aduna.iteration.CloseableIteration;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.hadoop.io.Text;
-import org.openrdf.query.Binding;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.QueryEvaluationException;
-import org.openrdf.query.algebra.evaluation.QueryBindingSet;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-import mvm.rya.indexing.PrecompQueryIndexer;
-import mvm.rya.indexing.external.tupleSet.AccumuloIndexSet.AccValueFactory;
-
-public class AccumuloPrecompQueryIndexer implements PrecompQueryIndexer {
-
-
- private Connector accCon;
- private String tableName;
- private Map<String, AccValueFactory> bindings;
-
-
-
- public AccumuloPrecompQueryIndexer(Connector accCon, String tableName) {
- this.accCon = accCon;
- this.tableName = tableName;
- }
-
-
- @Override
- public void storeBindingSet(BindingSet bs) throws IOException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void storeBindingSets(Collection<BindingSet> bindingSets) throws IOException, IllegalArgumentException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public CloseableIteration<BindingSet, QueryEvaluationException> queryPrecompJoin(List<String> varOrder,
- String localityGroup, Map<String, AccValueFactory> bindings, Map<String, org.openrdf.model.Value> valMap, Collection<BindingSet> bsConstraints)
- throws QueryEvaluationException, TableNotFoundException {
-
-
- final int prefixLength = Integer.parseInt(varOrder.remove(varOrder.size()-1));
- final Iterator<Entry<Key,Value>> accIter;
- final HashMultimap<Range,BindingSet> map = HashMultimap.create();
- final List<BindingSet> extProdList = Lists.newArrayList();
- final Map<String, AccValueFactory> bindingMap = bindings;
- final List<String> order = varOrder;
- final BatchScanner bs = accCon.createBatchScanner(tableName, new Authorizations(), 10);
- final Set<Range> ranges = Sets.newHashSet();
-
-
-
- bs.fetchColumnFamily(new Text(localityGroup));
-
- //process bindingSet and constant constraints
- for (BindingSet bSet : bsConstraints) {
- StringBuffer rangePrefix = new StringBuffer();
- int i = 0;
-
- for (String b : order) {
-
- if (i >= prefixLength) {
- break;
- }
-
- if (b.startsWith("-const-")) {
- String val = bindings.get(b).create(valMap.get(b));
- rangePrefix.append(val);
- rangePrefix.append("\u0000");
- } else {
-
- Binding v = bSet.getBinding(b);
- if (v == null) {
- throw new IllegalStateException("Binding set can't have null value!");
- }
- String val = bindings.get(b).create(bSet.getValue(b));
- rangePrefix.append(val);
- rangePrefix.append("\u0000");
-
- }
-
- i++;
-
- }
- if (rangePrefix.length() > 0) {
- String prefixWithOutNull = rangePrefix.deleteCharAt(rangePrefix.length() - 1).toString();
- String prefixWithNull = prefixWithOutNull + "\u0001";
- Range r = new Range(new Key(prefixWithOutNull), true, new Key(prefixWithNull), false);
- map.put(r, bSet);
- ranges.add(r);
- } else if (bSet.size() > 0) {
- extProdList.add(bSet);
- }
- }
-
- //constant constraints and no bindingSet constraints
- //add range of entire table if no constant constraints and
- //bsConstraints consists of single, empty set (occurs when AIS is
- //first node evaluated in query)
- if (ranges.isEmpty() && bsConstraints.size() > 0) {
-
- if (prefixLength > 0) {
- StringBuffer rangePrefix = new StringBuffer();
-
- int i = 0;
- for (String b : order) {
- if (i >= prefixLength) {
- break;
- }
- if (b.startsWith("-const-")) {
- String val = bindings.get(b).create(valMap.get(b));
- rangePrefix.append(val);
- rangePrefix.append("\u0000");
- }
- i++;
- }
-
- String prefixWithOutNull = rangePrefix.deleteCharAt(rangePrefix.length() - 1).toString();
- String prefixWithNull = prefixWithOutNull + "\u0001";
- Range r = new Range(new Key(prefixWithOutNull), true, new Key(prefixWithNull), false);
- ranges.add(r);
-
- } else { // no constant or bindingSet constraints
- ranges.add(new Range("", true, "~", false));
- }
- }
-
- if (ranges.size() == 0) {
- accIter = null;
- } else {
- bs.setRanges(ranges);
- accIter = bs.iterator();
- }
-
-
- return new CloseableIteration<BindingSet, QueryEvaluationException>() {
-
- @Override
- public void remove() throws QueryEvaluationException {
- throw new UnsupportedOperationException();
- }
-
- private Iterator<BindingSet> inputSet = null;
- private QueryBindingSet currentSolutionBs = null;
- private boolean hasNextCalled = false;
- private boolean isEmpty = false;
-
-
-
- @Override
- public BindingSet next() throws QueryEvaluationException {
- QueryBindingSet bs = new QueryBindingSet();
-
- if (hasNextCalled) {
- hasNextCalled = false;
- if (inputSet != null) {
- bs.addAll(inputSet.next());
- }
- bs.addAll(currentSolutionBs);
- } else if (isEmpty) {
- throw new NoSuchElementException();
- } else {
- if (this.hasNext()) {
- hasNextCalled = false;
- if (inputSet != null) {
- bs.addAll(inputSet.next());
- }
- bs.addAll(currentSolutionBs);
- } else {
- throw new NoSuchElementException();
- }
- }
-
- return bs;
- }
-
- @Override
- public boolean hasNext() throws QueryEvaluationException {
-
- if(accIter == null ) {
- isEmpty = true;
- return false;
- }
-
- if (!hasNextCalled && !isEmpty) {
- while (accIter.hasNext() || (inputSet != null && inputSet.hasNext())) {
-
- if(inputSet != null && inputSet.hasNext()) {
- hasNextCalled = true;
- return true;
- }
-
-
- Key k = accIter.next().getKey();
- final String[] s = k.getRow().toString().split("\u0000");
-
- StringBuilder rangePrefix = new StringBuilder();
- // TODO Assuming that order specifies order of variables
- // commmon to
- // bindingSet passed in and variables in index table
- // --size is equal to
-
- for (int i = 0; i < prefixLength; i++) {
- rangePrefix.append(s[i]);
- rangePrefix.append("\u0000");
- }
-
- // TODO I need to remember what the type was!
- currentSolutionBs = new QueryBindingSet();
- int i = 0;
- for (String b : order) {
- if (b.startsWith("-const")) {
- i++;
- } else {
- final String v = s[i];
- currentSolutionBs.addBinding(b, bindingMap.get(b).create(v));
- i++;
- }
-
- }
- //check to see if bindingSet constraints exist
- if (map.size() > 0) {
- String prefixWithOutNull = rangePrefix.deleteCharAt(rangePrefix.length() - 1).toString();
- String prefixWithNull = prefixWithOutNull + "\u0001";
- Range r = new Range(new Key(prefixWithOutNull), true, new Key(prefixWithNull), false);
- inputSet = map.get(r).iterator();
- if (!inputSet.hasNext()) {
- continue;
- } else {
- hasNextCalled = true;
- return true;
- } // check to see if binding set constraints exist, but no common vars
- } else if (extProdList.size() > 0) {
- inputSet = extProdList.iterator();
- hasNextCalled = true;
- return true;
- }else { //no bindingsSet constraints--only constant constraints or none
- hasNextCalled = true;
- return true;
- }
- }
-
- isEmpty = true;
- return false;
-
- } else if (isEmpty) {
- return false;
- } else {
- return true;
- }
-
- }
-
- @Override
- public void close() throws QueryEvaluationException {
- bs.close();
- }
-
- };
- }
-
-
-
- @Override
- public void flush() throws IOException {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void close() throws IOException {
- // TODO Auto-generated method stub
-
- }
-
-
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/DocIdIndexer.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/DocIdIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/DocIdIndexer.java
deleted file mode 100644
index 21d5de7..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/DocIdIndexer.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package mvm.rya.indexing;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import info.aduna.iteration.CloseableIteration;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Collection;
-
-import mvm.rya.indexing.accumulo.entity.StarQuery;
-
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.QueryEvaluationException;
-
-public interface DocIdIndexer extends Closeable {
-
-
-
- public abstract CloseableIteration<BindingSet, QueryEvaluationException> queryDocIndex(StarQuery query,
- Collection<BindingSet> constraints) throws TableNotFoundException, QueryEvaluationException;
-
-
-
- @Override
- public abstract void close() throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java b/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java
deleted file mode 100644
index 5d2678b..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/FilterFunctionOptimizer.java
+++ /dev/null
@@ -1,358 +0,0 @@
-package mvm.rya.indexing;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import mvm.rya.accumulo.AccumuloRdfConfiguration;
-import mvm.rya.indexing.IndexingFunctionRegistry.FUNCTION_TYPE;
-import mvm.rya.indexing.accumulo.ConfigUtils;
-import mvm.rya.indexing.accumulo.freetext.AccumuloFreeTextIndexer;
-import mvm.rya.indexing.accumulo.freetext.FreeTextTupleSet;
-import mvm.rya.indexing.accumulo.geo.GeoMesaGeoIndexer;
-import mvm.rya.indexing.accumulo.geo.GeoTupleSet;
-import mvm.rya.indexing.accumulo.temporal.AccumuloTemporalIndexer;
-import mvm.rya.indexing.accumulo.temporal.TemporalTupleSet;
-import mvm.rya.indexing.mongodb.MongoGeoIndexer;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.TableExistsException;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.commons.lang.Validate;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.geotools.feature.SchemaException;
-import org.openrdf.model.Resource;
-import org.openrdf.model.URI;
-import org.openrdf.model.Value;
-import org.openrdf.model.ValueFactory;
-import org.openrdf.model.impl.URIImpl;
-import org.openrdf.model.impl.ValueFactoryImpl;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.Dataset;
-import org.openrdf.query.algebra.And;
-import org.openrdf.query.algebra.Filter;
-import org.openrdf.query.algebra.FunctionCall;
-import org.openrdf.query.algebra.Join;
-import org.openrdf.query.algebra.LeftJoin;
-import org.openrdf.query.algebra.QueryModelNode;
-import org.openrdf.query.algebra.StatementPattern;
-import org.openrdf.query.algebra.TupleExpr;
-import org.openrdf.query.algebra.ValueConstant;
-import org.openrdf.query.algebra.ValueExpr;
-import org.openrdf.query.algebra.Var;
-import org.openrdf.query.algebra.evaluation.QueryOptimizer;
-import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
-
-import com.google.common.collect.Lists;
-
-public class FilterFunctionOptimizer implements QueryOptimizer, Configurable {
-
- private ValueFactory valueFactory = new ValueFactoryImpl();
-
- private Configuration conf;
- private GeoIndexer geoIndexer;
- private FreeTextIndexer freeTextIndexer;
- private TemporalIndexer temporalIndexer;
- private boolean init = false;
-
-
- public FilterFunctionOptimizer() {
- }
-
-
- public FilterFunctionOptimizer(AccumuloRdfConfiguration conf) throws AccumuloException, AccumuloSecurityException,
- TableNotFoundException, IOException, SchemaException, TableExistsException {
- this.conf = conf;
- init();
- }
-
- //setConf initializes FilterFunctionOptimizer so reflection can be used
- //to create optimizer in RdfCloudTripleStoreConnection
- @Override
- public void setConf(Configuration conf) {
- this.conf = conf;
- init();
- }
-
-
- private void init() {
- if (!init) {
- if (ConfigUtils.getUseMongo(conf)) {
- this.geoIndexer = new MongoGeoIndexer();
- geoIndexer.setConf(conf);
- } else {
- this.geoIndexer = new GeoMesaGeoIndexer();
- geoIndexer.setConf(conf);
- this.freeTextIndexer = new AccumuloFreeTextIndexer();
- freeTextIndexer.setConf(conf);
- this.temporalIndexer = new AccumuloTemporalIndexer();
- temporalIndexer.setConf(conf);
- init = true;
- }
- }
- }
-
- @Override
- public void optimize(TupleExpr tupleExpr, Dataset dataset, BindingSet bindings) {
- // find variables used in property and resource based searches:
- SearchVarVisitor searchVars = new SearchVarVisitor();
- tupleExpr.visit(searchVars);
- // rewrites for property searches:
- processPropertySearches(tupleExpr, searchVars.searchProperties);
-
- }
-
-
-
- private void processPropertySearches(TupleExpr tupleExpr, Collection<Var> searchProperties) {
- MatchStatementVisitor matchStatements = new MatchStatementVisitor(searchProperties);
- tupleExpr.visit(matchStatements);
- for (StatementPattern matchStatement: matchStatements.matchStatements) {
- Var subject = matchStatement.getSubjectVar();
- if (subject.hasValue() && !(subject.getValue() instanceof Resource))
- throw new IllegalArgumentException("Query error: Found " + subject.getValue() + ", expected an URI or BNode");
- Validate.isTrue(subject.hasValue() || subject.getName() != null);
- Validate.isTrue(!matchStatement.getObjectVar().hasValue() && matchStatement.getObjectVar().getName() != null);
- buildQuery(tupleExpr, matchStatement);
- }
-
- }
-
-
- private void buildQuery(TupleExpr tupleExpr, StatementPattern matchStatement) {
- //If our IndexerExpr (to be) is the rhs-child of LeftJoin, we can safely make that a Join:
- // the IndexerExpr will (currently) not return results that can deliver unbound variables.
- //This optimization should probably be generalized into a LeftJoin -> Join optimizer under certain conditions. Until that
- // has been done, this code path at least takes care of queries generated by OpenSahara SparqTool that filter on OPTIONAL
- // projections. E.g. summary~'full text search' (summary is optional). See #379
- if (matchStatement.getParentNode() instanceof LeftJoin) {
- LeftJoin leftJoin = (LeftJoin)matchStatement.getParentNode();
- if (leftJoin.getRightArg() == matchStatement && leftJoin.getCondition() == null)
- matchStatement.getParentNode().replaceWith(new Join(leftJoin.getLeftArg(), leftJoin.getRightArg()));
- }
- FilterFunction fVisitor = new FilterFunction(matchStatement.getObjectVar().getName());
- tupleExpr.visit(fVisitor);
- List<IndexingExpr> results = Lists.newArrayList();
- for(int i = 0; i < fVisitor.func.size(); i++){
- results.add(new IndexingExpr(fVisitor.func.get(i), matchStatement, fVisitor.args.get(i)));
- }
- removeMatchedPattern(tupleExpr, matchStatement, new IndexerExprReplacer(results));
-
- }
-
- //find vars contained in filters
- private static class SearchVarVisitor extends QueryModelVisitorBase<RuntimeException> {
-
- private final Collection<Var> searchProperties = new ArrayList<Var>();
-
- @Override
- public void meet(FunctionCall fn) {
- URI fun = new URIImpl(fn.getURI());
- Var result = IndexingFunctionRegistry.getResultVarFromFunctionCall(fun, fn.getArgs());
- if (result != null && !searchProperties.contains(result))
- searchProperties.add(result);
- }
- }
-
- //find StatementPatterns containing filter variables
- private static class MatchStatementVisitor extends QueryModelVisitorBase<RuntimeException> {
- private final Collection<Var> propertyVars;
- private final Collection<Var> usedVars = new ArrayList<Var>();
- private final List<StatementPattern> matchStatements = new ArrayList<StatementPattern>();
-
- public MatchStatementVisitor(Collection<Var> propertyVars) {
- this.propertyVars = propertyVars;
- }
-
- @Override public void meet(StatementPattern statement) {
- Var object = statement.getObjectVar();
- if (propertyVars.contains(object))
- if (usedVars.contains(object))
- throw new IllegalArgumentException("Illegal search, variable is used multiple times as object: " + object.getName());
- else {
- usedVars.add(object);
- matchStatements.add(statement);
- }
- }
- }
-
- private abstract class AbstractEnhanceVisitor extends QueryModelVisitorBase<RuntimeException> {
- final String matchVar;
- List<URI> func = Lists.newArrayList();
- List<Value[]> args = Lists.newArrayList();
-
- public AbstractEnhanceVisitor(String matchVar) {
- this.matchVar = matchVar;
- }
-
- protected void addFilter(URI uri, Value[] values) {
- func.add(uri);
- args.add(values);
- }
- }
-
- //create indexing expression for each filter matching var in filter StatementPattern
- //replace old filter condition with true condition
- private class FilterFunction extends AbstractEnhanceVisitor {
- public FilterFunction(String matchVar) {
- super(matchVar);
- }
-
- @Override
- public void meet(FunctionCall call) {
- URI fnUri = valueFactory.createURI(call.getURI());
- Var resultVar = IndexingFunctionRegistry.getResultVarFromFunctionCall(fnUri, call.getArgs());
- if (resultVar != null && resultVar.getName().equals(matchVar)) {
- addFilter(valueFactory.createURI(call.getURI()), extractArguments(matchVar, call));
- if (call.getParentNode() instanceof Filter || call.getParentNode() instanceof And || call.getParentNode() instanceof LeftJoin)
- call.replaceWith(new ValueConstant(valueFactory.createLiteral(true)));
- else
- throw new IllegalArgumentException("Query error: Found " + call + " as part of an expression that is too complex");
- }
- }
-
-
- private Value[] extractArguments(String matchName, FunctionCall call) {
- Value args[] = new Value[call.getArgs().size() - 1];
- int argI = 0;
- for (int i = 0; i != call.getArgs().size(); ++i) {
- ValueExpr arg = call.getArgs().get(i);
- if (argI == i && arg instanceof Var && matchName.equals(((Var)arg).getName()))
- continue;
- if (arg instanceof ValueConstant)
- args[argI] = ((ValueConstant)arg).getValue();
- else if (arg instanceof Var && ((Var)arg).hasValue())
- args[argI] = ((Var)arg).getValue();
- else
- throw new IllegalArgumentException("Query error: Found " + arg + ", expected a Literal, BNode or URI");
- ++argI;
- }
- return args;
- }
-
- @Override
- public void meet(Filter filter) {
- //First visit children, then condition (reverse of default):
- filter.getArg().visit(this);
- filter.getCondition().visit(this);
- }
- }
-
- private void removeMatchedPattern(TupleExpr tupleExpr, StatementPattern pattern, TupleExprReplacer replacer) {
- List<TupleExpr> indexTuples = replacer.createReplacement(pattern);
- if (indexTuples.size() > 1) {
- VarExchangeVisitor vev = new VarExchangeVisitor(pattern);
- tupleExpr.visit(vev);
- Join join = new Join(indexTuples.remove(0), indexTuples.remove(0));
- for (TupleExpr geo : indexTuples) {
- join = new Join(join, geo);
- }
- pattern.replaceWith(join);
- } else if (indexTuples.size() == 1) {
- pattern.replaceWith(indexTuples.get(0));
- pattern.setParentNode(null);
- } else {
- throw new IllegalStateException("Must have at least one replacement for matched StatementPattern.");
- }
- }
-
- private interface TupleExprReplacer {
- List<TupleExpr> createReplacement(TupleExpr org);
- }
-
-
- //replace each filter pertinent StatementPattern with corresponding index expr
- private class IndexerExprReplacer implements TupleExprReplacer {
- private final List<IndexingExpr> indxExpr;
- private FUNCTION_TYPE type;
-
- public IndexerExprReplacer(List<IndexingExpr> indxExpr) {
- this.indxExpr = indxExpr;
- URI func = indxExpr.get(0).getFunction();
- this.type = IndexingFunctionRegistry.getFunctionType(func);
- }
-
- @Override
- public List<TupleExpr> createReplacement(TupleExpr org) {
- List<TupleExpr> indexTuples = Lists.newArrayList();
- switch (type) {
- case GEO:
- for (IndexingExpr indx : indxExpr) {
- indexTuples.add(new GeoTupleSet(indx, geoIndexer));
- }
- break;
- case FREETEXT:
- for (IndexingExpr indx : indxExpr) {
- indexTuples.add(new FreeTextTupleSet(indx, freeTextIndexer));
- }
- break;
- case TEMPORAL:
- for (IndexingExpr indx : indxExpr) {
- indexTuples.add(new TemporalTupleSet(indx, temporalIndexer));
- }
- break;
- default:
- throw new IllegalArgumentException("Incorrect type!");
-
- }
- return indexTuples;
- }
- }
-
-
- private static class VarExchangeVisitor extends QueryModelVisitorBase<RuntimeException> {
-
- private final StatementPattern exchangeVar;
-
- public VarExchangeVisitor(StatementPattern sp) {
- this.exchangeVar = sp;
- }
-
- @Override
- public void meet(Join node) {
- QueryModelNode lNode = node.getLeftArg();
- if (lNode instanceof StatementPattern) {
- exchangeVar.replaceWith(lNode);
- node.setLeftArg(exchangeVar);
- } else {
- super.meet(node);
- }
- }
- }
-
-
-
-
-
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/FreeTextIndexer.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/FreeTextIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/FreeTextIndexer.java
deleted file mode 100644
index 2d8bae9..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/FreeTextIndexer.java
+++ /dev/null
@@ -1,62 +0,0 @@
-package mvm.rya.indexing;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import info.aduna.iteration.CloseableIteration;
-
-import java.io.IOException;
-import java.util.Set;
-
-import mvm.rya.api.persist.index.RyaSecondaryIndexer;
-
-import org.openrdf.model.Statement;
-import org.openrdf.model.URI;
-import org.openrdf.query.QueryEvaluationException;
-
-/**
- * A repository to store, index, and retrieve {@link Statement}s based on freetext features.
- */
-public interface FreeTextIndexer extends RyaSecondaryIndexer {
-
- /**
- * Query the Free Text Index with specific constraints. A <code>null</code> or empty parameters imply no constraint.
- *
- * @param query
- * the query to perform
- * @param contraints
- * the constraints on the statements returned
- * @return the set of statements that meet the query and other constraints.
- * @throws IOException
- */
- public abstract CloseableIteration<Statement, QueryEvaluationException> queryText(String query, StatementContraints contraints) throws IOException;
-
- /**
- * @return the set of predicates indexed by the indexer.
- */
- public abstract Set<URI> getIndexablePredicates();
-
- @Override
- public abstract void flush() throws IOException;
-
- @Override
- public abstract void close() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/GeoIndexer.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/GeoIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/GeoIndexer.java
deleted file mode 100644
index 7c04903..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/GeoIndexer.java
+++ /dev/null
@@ -1,201 +0,0 @@
-package mvm.rya.indexing;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import info.aduna.iteration.CloseableIteration;
-
-import java.io.IOException;
-import java.util.Set;
-
-import mvm.rya.api.persist.index.RyaSecondaryIndexer;
-
-import org.openrdf.model.Statement;
-import org.openrdf.model.URI;
-import org.openrdf.query.QueryEvaluationException;
-
-import com.vividsolutions.jts.geom.Geometry;
-
-/**
- * A repository to store, index, and retrieve {@link Statement}s based on geospatial features.
- */
-public interface GeoIndexer extends RyaSecondaryIndexer {
- /**
- * Returns statements that contain a geometry that is equal to the queried {@link Geometry} and meet the {@link StatementContraints}.
- *
- * <p>
- * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM):
- * <ul>
- * <li>
- * "Two geometries are topologically equal if their interiors intersect and no part of the interior or boundary of one geometry intersects the exterior of the other"
- * <li>"A is equal to B if A is within B and A contains B"
- * </ul>
- *
- * @param query
- * the queried geometry
- * @param contraints
- * the {@link StatementContraints}
- * @return
- */
- public abstract CloseableIteration<Statement, QueryEvaluationException> queryEquals(Geometry query, StatementContraints contraints);
-
- /**
- * Returns statements that contain a geometry that is disjoint to the queried {@link Geometry} and meet the {@link StatementContraints}.
- *
- * <p>
- * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM):
- * <ul>
- * <li>"A and B are disjoint if they have no point in common. They form a set of disconnected geometries."
- * <li>"A and B are disjoint if A does not intersect B"
- * </ul>
- *
- * @param query
- * the queried geometry
- * @param contraints
- * the {@link StatementContraints}
- * @return
- */
- public abstract CloseableIteration<Statement, QueryEvaluationException> queryDisjoint(Geometry query, StatementContraints contraints);
-
- /**
- * Returns statements that contain a geometry that Intersects the queried {@link Geometry} and meet the {@link StatementContraints}.
- *
- * <p>
- * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM):
- * <ul>
- * <li>"a intersects b: geometries a and b have at least one point in common."
- * <li>"not Disjoint"
- * </ul>
- *
- *
- * @param query
- * the queried geometry
- * @param contraints
- * the {@link StatementContraints}
- * @return
- */
- public abstract CloseableIteration<Statement, QueryEvaluationException> queryIntersects(Geometry query, StatementContraints contraints);
-
- /**
- * Returns statements that contain a geometry that Touches the queried {@link Geometry} and meet the {@link StatementContraints}.
- *
- * <p>
- * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM):
- * <ul>
- * <li>"a touches b, they have at least one boundary point in common, but no interior points."
- * </ul>
- *
- *
- * @param query
- * the queried geometry
- * @param contraints
- * the {@link StatementContraints}
- * @return
- */
- public abstract CloseableIteration<Statement, QueryEvaluationException> queryTouches(Geometry query, StatementContraints contraints);
-
- /**
- * Returns statements that contain a geometry that crosses the queried {@link Geometry} and meet the {@link StatementContraints}.
- *
- * <p>
- * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM):
- * <ul>
- * <li>
- * "a crosses b, they have some but not all interior points in common (and the dimension of the intersection is less than that of at least one of them)."
- * </ul>
- *
- * @param query
- * the queried geometry
- * @param contraints
- * the {@link StatementContraints}
- * @return
- */
- public abstract CloseableIteration<Statement, QueryEvaluationException> queryCrosses(Geometry query, StatementContraints contraints);
-
- /**
- * Returns statements that contain a geometry that is Within the queried {@link Geometry} and meet the {@link StatementContraints}.
- *
- * <p>
- * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM):
- * <ul>
- * <li>"a is within b, a lies in the interior of b"
- * <li>Same as: "Contains(b,a)"
- * </ul>
- *
- *
- * @param query
- * the queried geometry
- * @param contraints
- * the {@link StatementContraints}
- * @return
- */
- public abstract CloseableIteration<Statement, QueryEvaluationException> queryWithin(Geometry query, StatementContraints contraints);
-
- /**
- * Returns statements that contain a geometry that Contains the queried {@link Geometry} and meet the {@link StatementContraints}.
- *
- * <p>
- * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM):
- * <ul>
- * <li>b is within a. Geometry b lies in the interior of a. Another definition:
- * "a 'contains' b iff no points of b lie in the exterior of a, and at least one point of the interior of b lies in the interior of a"
- * <li>Same: Within(b,a)
- * </ul>
- *
- *
- * @param query
- * the queried geometry
- * @param contraints
- * the {@link StatementContraints}
- * @return
- */
- public abstract CloseableIteration<Statement, QueryEvaluationException> queryContains(Geometry query, StatementContraints contraints);
-
- /**
- * Returns statements that contain a geometry that Overlaps the queried {@link Geometry} and meet the {@link StatementContraints}.
- *
- * <p>
- * From Wikipedia (http://en.wikipedia.org/wiki/DE-9IM):
- * <ul>
- * <li>a crosses b, they have some but not all interior points in common (and the dimension of the intersection is less than that of at
- * least one of them).
- * </ul>
- *
- *
- * @param query
- * the queried geometry
- * @param contraints
- * the {@link StatementContraints}
- * @return
- */
- public abstract CloseableIteration<Statement, QueryEvaluationException> queryOverlaps(Geometry query, StatementContraints contraints);
-
- /**
- * @return the set of predicates indexed by the indexer.
- */
- public abstract Set<URI> getIndexablePredicates();
-
- @Override
- public abstract void flush() throws IOException;
-
- @Override
- public abstract void close() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/ExternalIndexMatcher.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/ExternalIndexMatcher.java b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/ExternalIndexMatcher.java
deleted file mode 100644
index ee3d444..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/ExternalIndexMatcher.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package mvm.rya.indexing.IndexPlanValidator;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import java.util.Iterator;
-
-import org.openrdf.query.algebra.TupleExpr;
-
-public interface ExternalIndexMatcher {
-
-
- public Iterator<TupleExpr> getIndexedTuples();
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/GeneralizedExternalProcessor.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/GeneralizedExternalProcessor.java b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/GeneralizedExternalProcessor.java
deleted file mode 100644
index 27a0d15..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/GeneralizedExternalProcessor.java
+++ /dev/null
@@ -1,730 +0,0 @@
-package mvm.rya.indexing.IndexPlanValidator;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import mvm.rya.indexing.external.QueryVariableNormalizer.VarCollector;
-import mvm.rya.indexing.external.tupleSet.ExternalTupleSet;
-
-import org.openrdf.query.algebra.BindingSetAssignment;
-import org.openrdf.query.algebra.Filter;
-import org.openrdf.query.algebra.Join;
-import org.openrdf.query.algebra.Projection;
-import org.openrdf.query.algebra.QueryModelNode;
-import org.openrdf.query.algebra.StatementPattern;
-import org.openrdf.query.algebra.TupleExpr;
-import org.openrdf.query.algebra.Var;
-import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
-import org.openrdf.query.algebra.helpers.StatementPatternCollector;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-/**
- * Processes a {@link TupleExpr} and replaces sets of elements in the tree with {@link ExternalTupleSet} objects.
- */
-public class GeneralizedExternalProcessor {
-
-
- /**
- * Iterates through list of normalized indexes and replaces all subtrees of query which match index with index.
- *
- * @param query
- * @return TupleExpr
- */
- public static TupleExpr process(TupleExpr query, List<ExternalTupleSet> indexSet) {
-
- boolean indexPlaced = false;
- TupleExpr rtn = query.clone();
-
-
- //TODO optimization: turn on when testing done
- QueryNodeCount qnc = new QueryNodeCount();
- rtn.visit(qnc);
-
- if(qnc.getNodeCount()/2 < indexSet.size()) {
- return null;
- }
-
-
- //move BindingSetAssignment Nodes out of the way
- organizeBSAs(rtn);
-
-
- // test to see if query contains no other nodes
- // than filter, join, projection, and statement pattern and
- // test whether query contains duplicate StatementPatterns and filters
- if (isTupleValid(rtn)) {
-
- for (ExternalTupleSet index : indexSet) {
-
- // test to see if index contains at least one StatementPattern,
- // that StatementPatterns are unique,
- // and that all variables found in filters occur in some
- // StatementPattern
- if (isTupleValid(index.getTupleExpr())) {
-
- ExternalTupleSet eTup = (ExternalTupleSet) index.clone();
- SPBubbleDownVisitor indexVistor = new SPBubbleDownVisitor(eTup);
- rtn.visit(indexVistor);
- FilterBubbleManager fbmv = new FilterBubbleManager(eTup);
- rtn.visit(fbmv);
- SubsetEqualsVisitor subIndexVis = new SubsetEqualsVisitor(eTup, rtn);
- rtn.visit(subIndexVis);
- indexPlaced = subIndexVis.indexPlaced();
- if(!indexPlaced) {
- break;
- }
-
- }
-
- }
- if(indexPlaced) {
-// if(indexSet.size() == 3) {
-// System.out.println("IndexSet is " + indexSet);
-// System.out.println("Tuple is " + rtn);
-// }
- return rtn;
- } else {
-// if(indexSet.size() == 3) {
-// System.out.println("IndexSet is " + indexSet);
-// }
-//
- return null;
- }
-
- } else {
- throw new IllegalArgumentException("Invalid Query.");
- }
- }
-
-
-
-
-
- // determines whether query is valid, which requires that a
- // query must contain a StatementPattern, not contain duplicate
- // Statement Patterns or Filters, not be comprised of only Projection,
- // Join, StatementPattern, and Filter nodes, and that any variable
- // appearing in a Filter must appear in a StatementPattern.
- private static boolean isTupleValid(QueryModelNode node) {
-
- ValidQueryVisitor vqv = new ValidQueryVisitor();
- node.visit(vqv);
-
- Set<String> spVars = getVarNames(getQNodes("sp", node));
-
- if (vqv.isValid() && (spVars.size() > 0)) {
-
- FilterCollector fvis = new FilterCollector();
- node.visit(fvis);
- List<QueryModelNode> fList = fvis.getFilters();
- return (fList.size() == Sets.newHashSet(fList).size() && getVarNames(fList).size() <= spVars.size());
-
- } else {
- return false;
- }
- }
-
- private static Set<QueryModelNode> getQNodes(QueryModelNode queryNode) {
- Set<QueryModelNode> rtns = new HashSet<QueryModelNode>();
-
- StatementPatternCollector spc = new StatementPatternCollector();
- queryNode.visit(spc);
- rtns.addAll(spc.getStatementPatterns());
-
- FilterCollector fvis = new FilterCollector();
- queryNode.visit(fvis);
- rtns.addAll(fvis.getFilters());
-
- ExternalTupleCollector eVis = new ExternalTupleCollector();
- queryNode.visit(eVis);
- rtns.addAll(eVis.getExtTup());
-
- return rtns;
- }
-
- private static Set<QueryModelNode> getQNodes(String node, QueryModelNode queryNode) {
-
- if (node.equals("sp")) {
- Set<QueryModelNode> eSet = new HashSet<QueryModelNode>();
- StatementPatternCollector spc = new StatementPatternCollector();
- queryNode.visit(spc);
- List<StatementPattern> spList = spc.getStatementPatterns();
- eSet.addAll(spList);
- // returns empty set if list contains duplicate StatementPatterns
- if (spList.size() > eSet.size()) {
- return Sets.newHashSet();
- } else {
- return eSet;
- }
- } else if (node.equals("filter")) {
-
- FilterCollector fvis = new FilterCollector();
- queryNode.visit(fvis);
-
- return Sets.newHashSet(fvis.getFilters());
- } else {
-
- throw new IllegalArgumentException("Invalid node type.");
- }
- }
-
- // moves StatementPatterns in query that also occur in index to bottom of
- // query tree.
- private static class SPBubbleDownVisitor extends QueryModelVisitorBase<RuntimeException> {
-
- private TupleExpr tuple;
- private QueryModelNode indexQNode;
- private Set<QueryModelNode> sSet = Sets.newHashSet();
-
- public SPBubbleDownVisitor(ExternalTupleSet index) {
-
- this.tuple = index.getTupleExpr();
- indexQNode = ((Projection) tuple).getArg();
- sSet = getQNodes("sp", indexQNode);
-
- }
-
- public void meet(Projection node) {
- // moves external tuples above statement patterns before attempting
- // to bubble down index statement patterns found in query tree
-
- organizeExtTuples(node);
-
- super.meet(node);
- }
-
- public void meet(Join node) {
- // if right node contained in index, move it to bottom of query tree
- if (sSet.contains(node.getRightArg())) {
-
- Set<QueryModelNode> eSet = getQNodes("sp", node);
- Set<QueryModelNode> compSet = Sets.difference(eSet, sSet);
-
- if (eSet.containsAll(sSet)) {
-
- QNodeExchanger qne = new QNodeExchanger(node.getRightArg(), compSet);
- node.visit(qne);
- node.replaceChildNode(node.getRightArg(), qne.getReplaced());
-
- super.meet(node);
- }
- return;
- }
- // if left node contained in index, move it to bottom of query tree
- else if (sSet.contains(node.getLeftArg())) {
-
- Set<QueryModelNode> eSet = getQNodes("sp", node);
- Set<QueryModelNode> compSet = Sets.difference(eSet, sSet);
-
- if (eSet.containsAll(sSet)) {
-
- QNodeExchanger qne = new QNodeExchanger(node.getLeftArg(), compSet);
- node.visit(qne);
- node.replaceChildNode(node.getLeftArg(), qne.getReplaced());
-
- super.meet(node);
- }
- return;
-
- } else {
- super.meet(node);
- }
-
- }
-
- // moves all ExternalTupleSets in query tree above remaining
- // StatementPatterns
- private static void organizeExtTuples(QueryModelNode node) {
-
- ExternalTupleCollector eVis = new ExternalTupleCollector();
- node.visit(eVis);
-
- ExtTupleExchangeVisitor oev = new ExtTupleExchangeVisitor(eVis.getExtTup());
- node.visit(oev);
- }
-
- }
-
- // given a replacement QueryModelNode and compSet, this visitor replaces the
- // first
- // element in the query tree that occurs in compSet with replacement and
- // returns
- // the element that was replaced.
- private static class QNodeExchanger extends QueryModelVisitorBase<RuntimeException> {
-
- private QueryModelNode toBeReplaced;
- private QueryModelNode replacement;
- private Set<QueryModelNode> compSet;
-
- public QNodeExchanger(QueryModelNode replacement, Set<QueryModelNode> compSet) {
- this.replacement = replacement;
- this.toBeReplaced = replacement;
- this.compSet = compSet;
- }
-
- public QueryModelNode getReplaced() {
- return toBeReplaced;
- }
-
- public void meet(Join node) {
-
- if (compSet.contains(node.getRightArg())) {
- this.toBeReplaced = node.getRightArg();
- node.replaceChildNode(node.getRightArg(), replacement);
- return;
- } else if (compSet.contains(node.getLeftArg())) {
- this.toBeReplaced = node.getLeftArg();
- node.replaceChildNode(node.getLeftArg(), replacement);
- return;
- } else {
- super.meet(node);
- }
-
- }
-
- }
-
- // moves filter that occurs in both query and index down the query tree so
- // that that it is positioned
- // above statement patterns associated with index. Precondition for calling
- // this method is that
- // SPBubbleDownVisitor has been called to position index StatementPatterns
- // within query tree.
- //TODO this visitor assumes that all filters are positioned at top of query tree
- //could lead to problems if filter optimizer called before external processor
- private static class FilterBubbleDownVisitor extends QueryModelVisitorBase<RuntimeException> {
-
- private QueryModelNode filter;
- private Set<QueryModelNode> compSet;
- private boolean filterPlaced = false;
-
- public FilterBubbleDownVisitor(QueryModelNode filter, Set<QueryModelNode> compSet) {
- this.filter = filter;
- this.compSet = compSet;
-
- }
-
- public boolean filterPlaced() {
- return filterPlaced;
- }
-
- public void meet(Join node) {
-
- if (!compSet.contains(node.getRightArg())) {
- // looks for placed to position filter node. if right node is
- // contained in index
- // and left node is statement pattern node contained in index or
- // is a join, place
- // filter above join.
- if (node.getLeftArg() instanceof Join || !(compSet.contains(node.getLeftArg()))) {
-
- QueryModelNode pNode = node.getParentNode();
- ((Filter) filter).setArg(node);
- pNode.replaceChildNode(node, filter);
- filterPlaced = true;
-
- return;
- } // otherwise place filter below join and above right arg
- else {
- ((Filter) filter).setArg(node.getRightArg());
- node.replaceChildNode(node.getRightArg(), filter);
- filterPlaced = true;
- return;
-
- }
- } else if ((node.getLeftArg() instanceof StatementPattern) && !compSet.contains(node.getLeftArg())) {
-
- ((Filter) filter).setArg(node.getLeftArg());
- node.replaceChildNode(node.getLeftArg(), filter);
- filterPlaced = true;
-
- return;
- } else {
- super.meet(node);
- }
- }
-
- }
-
- private static Set<String> getVarNames(Collection<QueryModelNode> nodes) {
-
- List<String> tempVars;
- Set<String> nodeVarNames = Sets.newHashSet();
-
- for (QueryModelNode s : nodes) {
- tempVars = VarCollector.process(s);
- for (String t : tempVars)
- nodeVarNames.add(t);
- }
- return nodeVarNames;
-
- }
-
- // visitor which determines whether or not to reposition a filter by calling
- // FilterBubbleDownVisitor
- private static class FilterBubbleManager extends QueryModelVisitorBase<RuntimeException> {
-
- private TupleExpr tuple;
- private QueryModelNode indexQNode;
- private Set<QueryModelNode> sSet = Sets.newHashSet();
- private Set<QueryModelNode> bubbledFilters = Sets.newHashSet();
-
- public FilterBubbleManager(ExternalTupleSet index) {
- this.tuple = index.getTupleExpr();
- indexQNode = ((Projection) tuple).getArg();
- sSet = getQNodes(indexQNode);
-
- }
-
- public void meet(Filter node) {
-
- Set<QueryModelNode> eSet = getQNodes(node);
- Set<QueryModelNode> compSet = Sets.difference(eSet, sSet);
-
- // if index contains filter node and it hasn't already been moved,
- // move it down
- // query tree just above position of statement pattern nodes found
- // in both query tree
- // and index (assuming that SPBubbleDownVisitor has already been
- // called)
- if (sSet.contains(node.getCondition()) && !bubbledFilters.contains(node.getCondition())) {
- FilterBubbleDownVisitor fbdv = new FilterBubbleDownVisitor((Filter) node.clone(), compSet);
- node.visit(fbdv);
- bubbledFilters.add(node.getCondition());
- // checks if filter correctly placed, and if it has been,
- // removes old copy of filter
- if (fbdv.filterPlaced()) {
-
- QueryModelNode pNode = node.getParentNode();
- TupleExpr cNode = node.getArg();
- pNode.replaceChildNode(node, cNode);
-
-
- super.meetNode(pNode);
- }
- super.meet(node);
-
- } else {
- super.meet(node);
- }
- }
- }
-
- // iterates through the query tree and attempts to match subtrees with
- // index. When a match is
- // found, the subtree is replaced by an ExternalTupleSet formed from the
- // index. Pre-condition for
- // calling this method is that both SPBubbleDownVisitor and
- // FilterBubbleManager have been called
- // to position the StatementPatterns and Filters.
- private static class SubsetEqualsVisitor extends QueryModelVisitorBase<RuntimeException> {
-
- private TupleExpr query;
- private TupleExpr tuple;
- private QueryModelNode indexQNode;
- private ExternalTupleSet set;
- private Set<QueryModelNode> sSet = Sets.newHashSet();
- private TupleExpr temp;
- private boolean indexPlaced = false;
-
-
- public SubsetEqualsVisitor(ExternalTupleSet index, TupleExpr query) {
- this.query = query;
- this.tuple = index.getTupleExpr();
- this.set = index;
- indexQNode = ((Projection) tuple).getArg();
- sSet = getQNodes(indexQNode);
-
- }
-
- public boolean indexPlaced() {
- return indexPlaced;
- }
-
-
- public void meet(Join node) {
-
- Set<QueryModelNode> eSet = getQNodes(node);
-
- if (eSet.containsAll(sSet) && !(node.getRightArg() instanceof BindingSetAssignment)) {
-
-// System.out.println("Eset is " + eSet + " and sSet is " + sSet);
-
- if (eSet.equals(sSet)) {
- node.replaceWith(set);
- indexPlaced = true;
- return;
- } else {
- if (node.getLeftArg() instanceof StatementPattern && sSet.size() == 1) {
- if(sSet.contains(node.getLeftArg())) {
- node.setLeftArg(set);
- indexPlaced = true;
- } else if(sSet.contains(node.getRightArg())) {
- node.setRightArg(set);
- indexPlaced = true;
- } else {
- return;
- }
- }
- else {
- super.meet(node);
- }
- }
- } else if (eSet.containsAll(sSet)) {
-
- super.meet(node);
-
- } else {
- return;
- }
-
- }
- //TODO might need to include BindingSetAssignment Condition here
- //to account for index consisting of only filter and BindingSetAssignment nodes
- public void meet(Filter node) {
-
- Set<QueryModelNode> eSet = getQNodes(node);
-
- if (eSet.containsAll(sSet)) {
-
- if (eSet.equals(sSet)) {
- node.replaceWith(set);
- indexPlaced = true;
- return;
- } else {
- node.getArg().visit(this);
- }
- }
- }
-
-
- public void meet(StatementPattern node) {
- return;
- }
- }
-
- // visitor which determines whether a query is valid (i.e. it does not
- // contain nodes other than
- // Projection, Join, Filter, StatementPattern )
- private static class ValidQueryVisitor extends QueryModelVisitorBase<RuntimeException> {
-
- private boolean isValid = true;
-
- public boolean isValid() {
- return isValid;
- }
-
- public void meet(Projection node) {
- node.getArg().visit(this);
- }
-
- public void meet(Filter node) {
- node.getArg().visit(this);
- }
-
-
-
-
-
- public void meetNode(QueryModelNode node) {
-
- if (!((node instanceof Join) || (node instanceof StatementPattern) || (node instanceof BindingSetAssignment) || (node instanceof Var))) {
- isValid = false;
- return;
-
- } else{
- super.meetNode(node);
- }
- }
-
- }
-
- // repositions ExternalTuples above StatementPatterns within query tree
- private static class ExtTupleExchangeVisitor extends QueryModelVisitorBase<RuntimeException> {
-
- private Set<QueryModelNode> extTuples;
-
- public ExtTupleExchangeVisitor(Set<QueryModelNode> extTuples) {
- this.extTuples = extTuples;
- }
-
- public void meet(Join queryNode) {
-
- // if query tree contains external tuples and they are not
- // positioned above statement pattern node
- // reposition
- if (this.extTuples.size() > 0 && !(queryNode.getRightArg() instanceof ExternalTupleSet)
- && !(queryNode.getRightArg() instanceof BindingSetAssignment)) {
-
- if (queryNode.getLeftArg() instanceof ExternalTupleSet) {
- QueryModelNode temp = queryNode.getLeftArg();
- queryNode.setLeftArg(queryNode.getRightArg());
- queryNode.setRightArg((TupleExpr)temp);
- } else {
-
- QNodeExchanger qnev = new QNodeExchanger((QueryModelNode) queryNode.getRightArg(), this.extTuples);
- queryNode.visit(qnev);
- queryNode.replaceChildNode(queryNode.getRightArg(), qnev.getReplaced());
- super.meet(queryNode);
- }
- } else {
- super.meet(queryNode);
- }
-
- }
-
- }
-
- private static class ExternalTupleCollector extends QueryModelVisitorBase<RuntimeException> {
-
- private Set<QueryModelNode> eSet = new HashSet<QueryModelNode>();
-
- @Override
- public void meetNode(QueryModelNode node) throws RuntimeException {
- if (node instanceof ExternalTupleSet) {
- eSet.add(node);
- }
- super.meetNode(node);
- }
-
- public Set<QueryModelNode> getExtTup() {
- return eSet;
- }
-
- }
-
- private static class FilterCollector extends QueryModelVisitorBase<RuntimeException> {
-
- private List<QueryModelNode> filterList = Lists.newArrayList();
-
- public List<QueryModelNode> getFilters() {
- return filterList;
- }
-
- @Override
- public void meet(Filter node) {
- filterList.add(node.getCondition());
- super.meet(node);
- }
-
- }
-
- private static void organizeBSAs(QueryModelNode node) {
-
- BindingSetAssignmentCollector bsac = new BindingSetAssignmentCollector();
- node.visit(bsac);
-
- if (bsac.containsBSAs()) {
- Set<QueryModelNode> bsaSet = bsac.getBindingSetAssignments();
- BindingSetAssignmentExchangeVisitor bsaev = new BindingSetAssignmentExchangeVisitor(bsaSet);
- node.visit(bsaev);
- }
- }
-
- // repositions ExternalTuples above StatementPatterns within query tree
- private static class BindingSetAssignmentExchangeVisitor extends QueryModelVisitorBase<RuntimeException> {
-
- private Set<QueryModelNode> bsas;
-
- public BindingSetAssignmentExchangeVisitor(Set<QueryModelNode> bsas) {
- this.bsas = bsas;
- }
-
- public void meet(Join queryNode) {
-
- // if query tree contains external tuples and they are not
- // positioned above statement pattern node
- // reposition
- if (this.bsas.size() > 0 && !(queryNode.getRightArg() instanceof BindingSetAssignment)) {
- QNodeExchanger qnev = new QNodeExchanger((QueryModelNode) queryNode.getRightArg(), bsas);
- queryNode.visit(qnev);
- queryNode.replaceChildNode(queryNode.getRightArg(), qnev.getReplaced());
- super.meet(queryNode);
- } else {
- super.meet(queryNode);
- }
-
- }
-
- }
-
-
- public static class BindingSetAssignmentCollector extends QueryModelVisitorBase<RuntimeException> {
-
- private Set<QueryModelNode> bindingSetList = Sets.newHashSet();
-
- public Set<QueryModelNode> getBindingSetAssignments() {
- return bindingSetList;
- }
-
- public boolean containsBSAs() {
- return (bindingSetList.size() > 0);
- }
-
- @Override
- public void meet(BindingSetAssignment node) {
- bindingSetList.add(node);
- super.meet(node);
- }
-
- }
-
-
-
- public static class QueryNodeCount extends QueryModelVisitorBase<RuntimeException> {
-
- private int nodeCount;
-
- public QueryNodeCount() {
- nodeCount = 0;
- }
-
- public int getNodeCount() {
- return nodeCount;
- }
-
-
- @Override
- public void meet(StatementPattern node) {
- nodeCount += 1;
- return;
- }
-
- @Override
- public void meet(Filter node) {
- nodeCount += 1;
- node.getArg().visit(this);
- }
-
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexListPruner.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexListPruner.java b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexListPruner.java
deleted file mode 100644
index fa1dc13..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexListPruner.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package mvm.rya.indexing.IndexPlanValidator;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import java.util.List;
-
-
-
-import java.util.Set;
-
-import mvm.rya.indexing.external.tupleSet.ExternalTupleSet;
-
-public interface IndexListPruner {
-
- public Set<ExternalTupleSet> getRelevantIndices(List<ExternalTupleSet> indexList);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexPlanValidator.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexPlanValidator.java b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexPlanValidator.java
deleted file mode 100644
index 74df958..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexPlanValidator.java
+++ /dev/null
@@ -1,210 +0,0 @@
-package mvm.rya.indexing.IndexPlanValidator;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-import mvm.rya.indexing.external.tupleSet.ExternalTupleSet;
-
-import org.openrdf.query.algebra.BindingSetAssignment;
-import org.openrdf.query.algebra.Filter;
-import org.openrdf.query.algebra.Join;
-import org.openrdf.query.algebra.Projection;
-import org.openrdf.query.algebra.StatementPattern;
-import org.openrdf.query.algebra.TupleExpr;
-import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
-
-import com.google.common.collect.Sets;
-
-
-
-
-public class IndexPlanValidator implements TupleValidator {
-
- private boolean omitCrossProd = false;
-
-
- public IndexPlanValidator(boolean omitCrossProd) {
- this.omitCrossProd = omitCrossProd;
- }
-
- public void setOmitCrossProd(boolean omitCrossProd) {
- this.omitCrossProd = omitCrossProd;
- }
-
-
- @Override
- public boolean isValid(TupleExpr te) {
-
- TupleValidateVisitor tv = new TupleValidateVisitor();
- te.visit(tv);
-
- return tv.isValid();
- }
-
-
-
-
- public int getValidTupleSize(Iterator<TupleExpr> iter) {
-
- int size = 0;
-
- while(iter.hasNext()) {
- if(isValid(iter.next())) {
- size++;
- }
- }
-
- return size;
-
- }
-
-
-
- @Override
- public Iterator<TupleExpr> getValidTuples(Iterator<TupleExpr> tupleIter) {
-
- final Iterator<TupleExpr> iter = tupleIter;
-
- return new Iterator<TupleExpr>() {
-
- private TupleExpr next = null;
- private boolean hasNextCalled = false;
- private boolean isEmpty = false;
-
- @Override
- public boolean hasNext() {
-
- if (!hasNextCalled && !isEmpty) {
- while (iter.hasNext()) {
- TupleExpr temp = iter.next();
- if (isValid(temp)) {
- next = temp;
- hasNextCalled = true;
- return true;
- }
- }
- isEmpty = true;
- return false;
- } else if(isEmpty) {
- return false;
- }else {
- return true;
- }
- }
-
- @Override
- public TupleExpr next() {
-
- if (hasNextCalled) {
- hasNextCalled = false;
- return next;
- } else if(isEmpty) {
- throw new NoSuchElementException();
- }else {
- if (this.hasNext()) {
- hasNextCalled = false;
- return next;
- } else {
- throw new NoSuchElementException();
- }
- }
- }
-
- @Override
- public void remove() {
-
- throw new UnsupportedOperationException("Cannot delete from iterator!");
-
- }
-
- };
- }
-
- private boolean isJoinValid(Join join) {
-
- Set<String> leftBindingNames = join.getLeftArg().getBindingNames();
- Set<String> rightBindingNames = join.getRightArg().getBindingNames();
-
-
- //System.out.println("Left binding names are " + leftBindingNames + " and right binding names are " + rightBindingNames);
-
- if (Sets.intersection(leftBindingNames, rightBindingNames).size() == 0) {
- if (omitCrossProd) {
- return false;
- } else {
- return true;
- }
-
- } else {
- if (join.getRightArg() instanceof ExternalTupleSet) {
-
- return ((ExternalTupleSet) join.getRightArg()).supportsBindingSet(leftBindingNames);
-
- } else {
- return true;
- }
- }
-
- }
-
- public class TupleValidateVisitor extends QueryModelVisitorBase<RuntimeException> {
-
- private boolean isValid = true;
-
- public boolean isValid() {
- return isValid;
- }
-
- @Override
- public void meet(Projection node) {
- node.getArg().visit(this);
- }
-
- @Override
- public void meet(StatementPattern node) {
- return;
- }
-
- public void meet(BindingSetAssignment node) {
- return;
- }
-
- @Override
- public void meet(Filter node) {
- node.getArg().visit(this);
- }
-
- @Override
- public void meet(Join node) {
- if (isJoinValid(node)) {
- super.meet(node);
- } else {
- isValid = false;
- return;
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexTupleGenerator.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexTupleGenerator.java b/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexTupleGenerator.java
deleted file mode 100644
index 3586a5e..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/IndexPlanValidator/IndexTupleGenerator.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package mvm.rya.indexing.IndexPlanValidator;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import java.util.Iterator;
-
-import org.openrdf.query.algebra.TupleExpr;
-
-public interface IndexTupleGenerator {
-
-
- public Iterator<TupleExpr> getPlans(Iterator<TupleExpr> indexPlans);
-
-
-}