You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@rya.apache.org by mi...@apache.org on 2015/12/07 13:04:55 UTC
[25/51] [partial] incubator-rya git commit: Cannot delete temp branch,
doc'd it.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/external/QueryVariableNormalizer.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/QueryVariableNormalizer.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/QueryVariableNormalizer.java
deleted file mode 100644
index d19c511..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/external/QueryVariableNormalizer.java
+++ /dev/null
@@ -1,1180 +0,0 @@
-package mvm.rya.indexing.external;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-import org.openrdf.model.Literal;
-import org.openrdf.model.Value;
-import org.openrdf.model.impl.ValueFactoryImpl;
-import org.openrdf.query.algebra.Filter;
-import org.openrdf.query.algebra.NAryValueOperator;
-import org.openrdf.query.algebra.ProjectionElem;
-import org.openrdf.query.algebra.ProjectionElemList;
-import org.openrdf.query.algebra.QueryModelNode;
-import org.openrdf.query.algebra.StatementPattern;
-import org.openrdf.query.algebra.TupleExpr;
-import org.openrdf.query.algebra.ValueConstant;
-import org.openrdf.query.algebra.ValueExpr;
-import org.openrdf.query.algebra.Var;
-import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-public class QueryVariableNormalizer {
-
-
- /**
- * @param tuple1
- * tuple expression from a parsed query
- * @param tuple2
- * tuple expression from a parsed query (the proposed index whose
- * variables are to be relabeled)
- * @return list of all possible tuples obtained by substituting the
- * variables of proposed index with the variables from query
- * @throws Exception
- * @throws IllegalArgumentException
- */
- public static List<TupleExpr> getNormalizedIndex(TupleExpr tuple1, TupleExpr tuple2) throws Exception {
-
- List<QueryModelNode> nodes1, nodes2;
- TreeMap<String, List<QueryModelNode>> queryMap1, indexMap1;
- List<HashMap<String, String>> varChanges = new ArrayList<HashMap<String, String>>();
- List<TupleExpr> tupleList = new ArrayList<TupleExpr>();
-
-
-
- // if tuples are equal, no need to do anything
- if (tuple1.equals(tuple2)) {
- tupleList.add((TupleExpr) tuple1.clone());
- return tupleList;
- }
-
-
- NormalizeQueryVisitor tupNVis = new NormalizeQueryVisitor(false);
- NormalizeQueryVisitor indexNVis = new NormalizeQueryVisitor(true);
- tuple1.visit(tupNVis);
- tuple2.visit(indexNVis);
-
-
- TupleExpr tuple;
- queryMap1 = tupNVis.getMap();
- indexMap1 = indexNVis.getMap();
-
- // TreeMaps that used for comparators
- TreeMap<String, Integer>[] trees = (TreeMap<String, Integer>[]) new TreeMap[4];
- for (int i = 0; i < 4; i++) {
- trees[i] = new TreeMap<String, Integer>();
- }
-
- trees[0] = tupNVis.getKeyMap(); // query tuple variable count
- trees[2] = indexNVis.getKeyMap(); // index tuple variable count
-
-
- // if query does not contain as many constant Vars as index,
- // normalization not possible.
-// if (!(trees[0].keySet().size() >= trees[2].keySet().size())) {
-// System.out.println("In here:1");
-// return tupleList;
-// }
-
- // sort keys according to size of associated StatementPattern list
- // this optimization ensures that initial list of HashMaps (possible
- // variable substitutions)
- // is as small as possible
- // Maybe add additional criteria to comparator taking into account size
- // of query bin lists
- Set<String> keys = indexMap1.keySet();
- List<String> keyList = new ArrayList<String>(keys);
- Collections.sort(keyList, new ConstantKeyComp(indexMap1, queryMap1));
-
- // iterate through constant values associated with smaller tuple,
- // check that larger tuple constants these constants, and use lists
- // of associated statement patterns to begin to construct variable
- // substitutions
- // that are consistent
-
- for (String s : keyList) {
- if (queryMap1.containsKey(s)) {
- nodes1 = queryMap1.get(s);
- nodes2 = indexMap1.get(s);
-
-
- if (!(nodes1.size() >= nodes2.size())) {
-// System.out.println("In here: 2");
-// System.out.println("Node lists are " + nodes1 + " and " +
-// nodes2);
- return tupleList;
- }
-
- trees[1] = getListVarCnt(nodes1, tupNVis.getVariableMap()); // query
- // list
- // variable
- // count
- trees[3] = getListVarCnt(nodes2, indexNVis.getVariableMap()); // index
- // list
- // variable
- // count
- Collections.sort(nodes1, new CountComp(trees[1], trees[0]));
- Collections.sort(nodes2, new CountComp(trees[3], trees[2]));
-
- varChanges = statementCompare(nodes1, nodes2, varChanges, trees);
-
- if (varChanges.size() == 0) {
- return tupleList;
- }
- }
-
- else {
- return tupleList;
- }
-
- }
-
- List<QueryModelNode> filters2 = indexNVis.getFilters();
- // determine if index contains filters whose variables need to be relabeled
- if (filters2.size() != 0) {
- List<QueryModelNode> filters1 = tupNVis.getFilters();
- // only attempt to normalize variables if query contains more filters than index
- if (filters1.size() >= filters2.size()) {
- Collections.sort(filters1, new FilterComp());
- Collections.sort(filters2, new FilterComp());
-
- varChanges = statementCompare(filters1, filters2, varChanges, trees);
-
- }
- }
-
- List<HashMap<String, String>> varChangeSet = new ArrayList<HashMap<String, String>>();
-
- for (HashMap<String, String> s : varChanges) {
- if (!varChangeSet.contains(s)) {
- varChangeSet.add(s);
- }
-
- }
-
-
- ValueMapVisitor valMapVis = new ValueMapVisitor();
- tuple1.visit(valMapVis);
- Map<String, Value> valMap = valMapVis.getValueMap();
-
-
- for (HashMap<String, String> s : varChangeSet) {
- //System.out.println(s);
- tuple = tuple2.clone();
- replaceTupleVariables(s, tuple, valMap);
- tupleList.add(tuple);
- }
-
- return tupleList;
-
- }
-
- /**
- * Produces a list of all possible substitutions stored in HashMaps that are
- * consistent with the two lists of statement patterns
- *
- * @param qArray
- * list of Statement nodes from query tuple
- * @param iArray
- * list of Statement nodes from index tuple
- * @param hMaps
- * HashMap containing variable substitutions
- * @param trees
- * TreeMaps used for statement pattern node ordering
- * @return
- */
- private static List<HashMap<String, String>> statementCompare(List<QueryModelNode> qArray,
- List<QueryModelNode> iArray, List<HashMap<String, String>> hMaps, TreeMap<String, Integer>[] trees) {
-
- if (hMaps.size() == 0) {
- HashMap<HashMap<String, String>, Boolean> mapConsistent = new HashMap<HashMap<String, String>, Boolean>();
- HashMap<String, String> hMap = new HashMap<String, String>();
- mapConsistent.put(hMap, false);
- evaluateMap(qArray, iArray, hMap, hMaps, mapConsistent, trees);
-
- return hMaps;
- }
-
- else {
-
- ArrayList<HashMap<String, String>> tempMaps = Lists.newArrayList(hMaps);
- HashMap<HashMap<String, String>, Boolean> mapConsistent = new HashMap<HashMap<String, String>, Boolean>();
- for (HashMap<String, String> s : hMaps) {
- mapConsistent.put(s, false);
- }
- for (HashMap<String, String> s : hMaps) {
- evaluateMap(qArray, iArray, s, tempMaps, mapConsistent, trees);
- }
-
- return tempMaps;
-
- }
- }
-
-
- /**
- * Adds or removes HashMap substitution schemes to the list of substitutions
- * schemes depending on whether or not they are consistent with the two
- * lists of statement patterns
- *
- * @param qArray
- * List of StatementPatterns associated with query array
- * @param iArray
- * List of StatementPatterns associated with index array
- * @param hMap
- * HashMap of substitutions to be analyzed for consistent and
- * added or removed
- * @param hMaps
- * List of HashMaps containing substitution schemes
- * @param trees
- * Array of TreeMaps used for comparison of StatementPattern
- * nodes
- */
- private static void evaluateMap(List<QueryModelNode> qArray, List<QueryModelNode> iArray,
- HashMap<String, String> hMap, List<HashMap<String, String>> hMaps,
- HashMap<HashMap<String, String>, Boolean> mapConsistent, TreeMap<String, Integer>[] trees) throws IllegalArgumentException {
-
- // if all nodes in indexArray have been exhausted, add map of substitutions to
- // list of possible substitution schemes.
- if (iArray.size() == 0) {
- if (!hMaps.contains(hMap)) {
- hMaps.add(hMap);
- }
- mapConsistent.put(hMap, true);
- return;
- }
-
- // for a given constant key, iterate through possible combinations of statement pattern nodes contained in associated query list and
- // index list to generate all possible substitution schemes.
- for (int i = 0; i < iArray.size(); i++) {
- for (int j = 0; j < qArray.size(); j++) {
- //System.out.println("Query list is " + qArray+ " and index list is " + iArray);
-
- QueryModelNode node1 = qArray.get(j);
- QueryModelNode node2 = iArray.get(i);
- // if lists contain statement patterns, check to see if two given statement patterns have same structure
- // independent of variables names (same constants in same place, non constant Vars in same place)
- if ((node1 instanceof StatementPattern) && (node2 instanceof StatementPattern)) {
- if (genConstantCompare((StatementPattern) node1, (StatementPattern) node2)) {
-
- List<Var> variables1 = ((StatementPattern)node1).getVarList();
- List<Var> variables2 = ((StatementPattern)node2).getVarList();
-
- List<List<String>> vars = genGetCommonVars(variables1, variables2);
- List<String> vars1 = vars.get(0);
- List<String> vars2 = vars.get(1);
-
- if (listConsistent(vars1, vars2, hMap)) {
-
- HashMap<String, String> hashMap = Maps.newHashMap(hMap);
- putVars(vars1, vars2, hashMap);
-
- List<QueryModelNode> queryArray = Lists.newArrayList(qArray);
- List<QueryModelNode> indexArray = Lists.newArrayList(iArray);
-
- indexArray.remove(i);
- queryArray.remove(j);
-
- evaluateMap(queryArray, indexArray, hashMap, hMaps, mapConsistent, trees);
- }
- }
- } // if lists contain filters, see if filters have same structure independent of variables names
- //(check that conditions are same independent of variable names).
- else if ((node1 instanceof Filter) && (node2 instanceof Filter)) {
- try {
- if (filterCompare((Filter) node1, (Filter) node2)) {
-
- List<QueryModelNode> variables1 = FilterVarValueCollector.process(((Filter) node1).getCondition());
- List<QueryModelNode> variables2 = FilterVarValueCollector.process(((Filter) node2).getCondition());
-
- List<List<String>> vars = filterCommonVars(variables1, variables2);
- List<String> vars1 = vars.get(0);
- List<String> vars2 = vars.get(1);
-
- if (listConsistent(vars1, vars2, hMap)) {
-
- HashMap<String, String> hashMap = Maps.newHashMap(hMap);
- putVars(vars1, vars2, hashMap);
-
- List<QueryModelNode> queryArray = Lists.newArrayList(qArray);
- List<QueryModelNode> indexArray = Lists.newArrayList(iArray);
-
- indexArray.remove(i);
- queryArray.remove(j);
-
- evaluateMap(queryArray, indexArray, hashMap, hMaps, mapConsistent, trees);
- }
-
- }
- } catch (Exception e) {
- System.out.println("Invalid Filter! " + e);
- }
-
- } else {
- throw new IllegalArgumentException("Invalid query tree.");
- }
-
- }
- }
- if (mapConsistent.containsKey(hMap))
- if (mapConsistent.get(hMap) == false) {
- hMaps.remove(hMap);
- }
- return;
-
- }
-
-
-
-
- private static List<List<String>> genGetCommonVars(List<Var> vars1, List<Var> vars2) {
-
-
- List<List<String>> varList = Lists.newArrayList();
- List<String> varList1 = Lists.newArrayList();
- List<String> varList2 = Lists.newArrayList();
-
-
-
- for (int i = 0; i < vars1.size(); i++) {
-
- if (!vars1.get(i).isConstant() && !vars2.get(i).isConstant()) {
-
- varList1.add(vars1.get(i).getName());
- varList2.add(vars2.get(i).getName());
-
- } else if(vars1.get(i).isConstant() && !vars2.get(i).isConstant()) {
- varList1.add(vars1.get(i).getName());
- varList2.add(vars2.get(i).getName());
- }
-
- }
-
- varList.add(varList1);
- varList.add(varList2);
-
- return varList;
- }
-
-
- private static List<List<String>> filterCommonVars(List<QueryModelNode> vars1, List<QueryModelNode> vars2) {
-
-
- List<List<String>> varList = Lists.newArrayList();
- List<String> varList1 = Lists.newArrayList();
- List<String> varList2 = Lists.newArrayList();
-
-
-
- for (int i = 0; i < vars1.size(); i++) {
-
- if ((vars1.get(i) instanceof ValueConstant) && (vars2.get(i) instanceof Var)) {
-
- ValueConstant vc = (ValueConstant) vars1.get(i);
- String s = vc.getValue().toString();
- if(vc.getValue() instanceof Literal) {
- s = s.substring(1, s.length() - 1);
- }
- s = "-const-" + s;
- varList1.add(s);
- varList2.add(((Var)vars2.get(i)).getName());
- } else if(!(vars1.get(i) instanceof ValueConstant)){
- if (!((Var) vars1.get(i)).isConstant() && (vars2.get(i) instanceof Var)
- && !((Var) vars2.get(i)).isConstant()) {
- varList1.add(((Var) vars1.get(i)).getName());
- varList2.add(((Var) vars2.get(i)).getName());
- } else if (((Var) vars1.get(i)).isConstant() && (vars2.get(i) instanceof Var)
- && !((Var) vars2.get(i)).isConstant()) {
- varList1.add(((Var) vars1.get(i)).getName());
- varList2.add(((Var) vars2.get(i)).getName());
- }
- }
-
- }
-
- varList.add(varList1);
- varList.add(varList2);
-
- return varList;
- }
-
-
-
- private static boolean genConstantCompare(StatementPattern queryNode, StatementPattern indexNode) {
-
-
-
- ArrayList<Var> vars1 = (ArrayList<Var>) queryNode.getVarList();
- ArrayList<Var> vars2 = (ArrayList<Var>) indexNode.getVarList();
-
-
- for (int i = 0; i < vars1.size(); i++) {
-
- if (vars1.get(i).isConstant() && vars2.get(i).isConstant()) {
-
- if (!vars1.get(i).equals(vars2.get(i))) {
- return false;
-
- }
-
- } else if(!vars1.get(i).isConstant() && vars2.get(i).isConstant() ) {
- return false;
- }
-
- }
-
- return true;
-
- }
-
-
-
-
- /**
- * Method checks that substituting val for key is consistent with
- * substitutions in hMap
- *
- * @param val
- * substituting variable
- * @param key
- * variable to be substituted for
- * @param hMap
- * HashMap containing the substitutions to be made
- * @return true if the proposed substitution is consistent with hMap, and
- * false otherwise
- */
- private static boolean checkVariables(String val, String key, HashMap<String, String> hMap) {
-
- if (!hMap.containsKey(key) && !hMap.containsValue(val)) {
-
- return true;
- } else if (!hMap.containsKey(key) && hMap.containsValue(val) || hMap.containsKey(key)
- && !hMap.containsValue(val)) {
-
- return false;
- } else {
-
- if (hMap.get(key).equals(val)) {
- return true;
- } else
- return false;
-
- }
-
- }
-
-
-
-
-
-
- // given two lists of variables and a HashMap, checks to see if substituting variable names in varList1
- // for variable names in varList2 is consistent with map.
- private static boolean listConsistent(List<String> varList1, List<String> varList2, HashMap<String, String> hMap) {
-
- for (int k = 0; k < varList1.size(); k++) {
-
- String s1 = varList1.get(k);
- String s2 = varList2.get(k);
- if (!checkVariables(s1, s2, hMap)) {
- return false;
- }
- }
- return true;
-
- }
-
-
- // given two lists of variables and a HashMap, substitutes variable names in varList1
- // for variable names in varList2 by updating map.
- private static void putVars(List<String> varList1, List<String> varList2, HashMap<String, String> hashMap) {
-
- for (int k = 0; k < varList1.size(); k++) {
- String s1 = varList1.get(k);
- String s2 = varList2.get(k);
- if (!hashMap.containsKey(s2)) {
-
- hashMap.put(s2, s1);
- }
- }
-
- }
-
-
- /**
- * @param filter1
- * @param filter2
- * @return true if filter2 is equal to filter1 once variables in filter2 are replaced with variables and constants
- * occurring in same position in filter1 (allows filter1 to contain constants where filter2 contains variables)
- * @throws Exception
- */
- private static boolean filterCompare(Filter filter1, Filter filter2) throws Exception {
-
- NodeCollector nc1 = new NodeCollector();
- NodeCollector nc2 = new NodeCollector();
-
- filter1.getCondition().visit(nc1);
- filter2.getCondition().visit(nc2);
-
- List<QueryModelNode> nodeList1 = nc1.getNodes();
- List<QueryModelNode> nodeList2 = nc2.getNodes();
-
- if (nodeList1.size() != nodeList2.size()) {
- return false;
- }
-
- for (int i = 0; i < nodeList1.size(); i++) {
- if ((nodeList1.get(i) instanceof ValueConstant) && (nodeList2.get(i) instanceof Var)) {
- continue;
- } else {
- if (nodeList1.get(i).getClass() != nodeList2.get(i).getClass()) {
- return false;
- }
- }
- }
-
- return true;
-
- }
-
- /**
- * Given a HashMap containing variable substitutions and a tuple, this
- * method uses a visitor to iterate through the tuple and make the necessary
- * substitutions
- *
- * @param varChanges
- * @param tuple
- * @throws Exception
- */
- private static void replaceTupleVariables(HashMap<String, String> varChanges, TupleExpr tuple, Map<String,Value> valMap) throws Exception {
-
- TupleVarRenamer visitor = new TupleVarRenamer(varChanges, valMap);
- tuple.visit(visitor);
- }
-
- /**
- * Given a list of StatementPattern nodes and a TreeMap containing the
- * variables in the tuple, this method counts the number of occurrences of
- * each variable in the given list
- *
- * @param list
- * List of StatementPattern nodes
- * @param cnt
- * TreeMap whose keys are tuple variables and whose value is 0
- * @return TreeMap whose keys are tuple variables and whose value is the
- * number of times variable appears in list
- */
- private static TreeMap<String, Integer> getListVarCnt(List<QueryModelNode> list, TreeMap<String, Integer> cnt) {
-
- int count = 0;
-
- for (QueryModelNode qNode : list) {
- List<String> vars = VarCollector.process(qNode);
- for (String s : vars) {
- count = cnt.get(s);
- count++;
- cnt.put(s, count);
- }
-
- }
-
- return cnt;
-
- }
-
- /**
- * Given a StatementPattern and two TreeMaps containing the variable counts
- * associated with an associated list and tuple, this method assigns a
- * number to the StatementPattern node which is determined by the number of
- * times its variables (non-constant Vars) appear in the list and throughout
- * the tuple
- *
- * @param sp
- * StatementPattern node
- * @param listCount
- * TreeMap with variable count info associated with list
- * @param tupCount
- * TreeMap with variable count info associated with tuple
- * @return count info associated with StatementPattern node
- */
- private static int getSpCount(QueryModelNode sp, TreeMap<String, Integer> listCount,
- TreeMap<String, Integer> tupCount) {
-
- int spCount = 0;
-
- List<String> vars = VarCollector.process(sp);
- for (String var : vars) {
- spCount = spCount + listCount.get(var) + tupCount.get(var);
- }
- return spCount;
-
- }
-
- /**
- * @return NormalizedQueryVisitor
- */
- public static NormalizeQueryVisitor getVisitor(boolean isIndex) {
- return new NormalizeQueryVisitor(isIndex);
-
- }
-
-
- // ********************Definition of Comparators****************
- // *************************************************************
- public static class CountComp implements Comparator<QueryModelNode> {
-
- private TreeMap<String, Integer> lCount, tupleCount;
-
- public CountComp(TreeMap<String, Integer> lCount, TreeMap<String, Integer> tupleCount) {
-
- this.lCount = lCount;
- this.tupleCount = tupleCount;
- }
-
- // compares StatementPattern nodes based on frequency at which their
- // variables appear in other StatementPattern nodes in associated
- // tuple and list
-
- public int compare(QueryModelNode sp1, QueryModelNode sp2) {
-
- return -(getSpCount(sp1, lCount, tupleCount) - getSpCount(sp2, lCount, tupleCount));
- }
-
- }
-
- // comparator to sort constant key list according to size of associated
- // StatementPattern array
- public static class ConstantKeyComp implements Comparator<String> {
-
- private TreeMap<String, List<QueryModelNode>> indexMap, queryMap;
-
- public ConstantKeyComp(TreeMap<String, List<QueryModelNode>> indexMap,
- TreeMap<String, List<QueryModelNode>> queryMap) {
-
- this.indexMap = indexMap;
- this.queryMap = queryMap;
-
- }
-
- // Compare method to sort keys of HashMap<String,
- // ArrayList<StatementPattern>
- // for index based on whether key also appears in query Map--if key does
- // not appear
- // in query map, key is given value 0 so it is moved to front when key
- // list is sorted.
- // If key appears in query map, key is assigned value that is the sum of
- // the size of the associated
- // lists in index map and query map.
-
- public int compare(String key1, String key2) {
-
- int len1 = 0;
- int len2 = 0;
-
- if (queryMap.containsKey(key1) && indexMap.containsKey(key1))
- len1 = indexMap.get(key1).size() + queryMap.get(key1).size();
- if (queryMap.containsKey(key2) && indexMap.containsKey(key2))
- len2 = indexMap.get(key2).size() + queryMap.get(key2).size();
-
- return (len1 - len2);
-
- }
-
- }
-
- public static class FilterComp implements Comparator<QueryModelNode> {
-
- public int compare(QueryModelNode q1, QueryModelNode q2) {
-
- int size1 = VarCollector.process(q1).size();
- int size2 = VarCollector.process(q2).size();
-
- return size1 - size2;
-
- }
-
- }
-
- // ******************** Definition of Visitors*****************
- // ************************************************************
-
-
-
- public static class ValueMapVisitor extends QueryModelVisitorBase<Exception> {
-
-
- private Map<String, Value> valMap = Maps.newHashMap();
-
-
-
- public Map<String, Value> getValueMap() {
- return valMap;
- }
-
- public void meet(Var var) {
- if (var.isConstant()) {
- valMap.put(var.getName(),var.getValue());
- }
-
-
- }
-
- public void meet(ValueConstant val) {
-
- String s = val.getValue().toString();
-
- if (val.getValue() instanceof Literal) {
- s = s.substring(1, s.length() - 1);
- }
-
- s = "-const-" + s;
- valMap.put(s, val.getValue());
- }
-
- }
-
-
-
-
-
-
-
- public static class NodeCollector extends QueryModelVisitorBase<Exception> {
-
-
- private List<QueryModelNode> nodes = Lists.newArrayList();
-
- public List<QueryModelNode> getNodes() {
- return nodes;
- }
-
- @Override
- public void meetNode(QueryModelNode node) throws Exception {
- nodes.add(node);
- super.meetNode(node);
- }
-
- }
-
- public static class SpVarReNamer extends QueryModelVisitorBase<RuntimeException> {
-
- private final HashMap<String, String> hMap;
- private Map<String, Value> valMap;
- private final ValueFactoryImpl vf = new ValueFactoryImpl();
-
- public SpVarReNamer(HashMap<String, String> hMap, Map<String, Value> valMap) {
- this.valMap = valMap;
- this.hMap = hMap;
- }
-
- public void meet(Var var) {
- if (!var.isConstant() && hMap.containsKey(var.getName())) {
- String val = hMap.get(var.getName());
- if (val.startsWith("-const-")) {
- var.setName(val);
- var.setValue(valMap.get(val));
- var.setAnonymous(true); //TODO this might be a hack -- when are Vars not anonymous?
- } else {
- var.setName(val);
- }
- }
- }
-
- }
-
-
-
-
- public static class FilterVarReNamer extends QueryModelVisitorBase<RuntimeException> {
-
- private final HashMap<String, String> hMap;
- private Map<String, Value> valMap;
- private final ValueFactoryImpl vf = new ValueFactoryImpl();
-
- public FilterVarReNamer(HashMap<String, String> hMap, Map<String, Value> valMap) {
- this.valMap = valMap;
- this.hMap = hMap;
- }
-
- @Override
- public void meet(Var var) {
-
- if (!(var.getParentNode() instanceof NAryValueOperator)) {
- if (!var.isConstant() && hMap.containsKey(var.getName())) {
- String val = hMap.get(var.getName());
- if (val.startsWith("-const-")) {
- var.replaceWith(new ValueConstant(valMap.get(val)));
- } else {
- var.setName(val);
- }
- }
- }
- }
-
-
-
- @Override
- public void meetNAryValueOperator(NAryValueOperator node) {
-
- List<ValueExpr> oldValues = node.getArguments();
- List<ValueExpr> newValues = Lists.newArrayList();
-
- for (ValueExpr v : oldValues) {
- if (v instanceof Var) {
- Var var = (Var) v;
- if (!(var.isConstant() && hMap.containsKey(var.getName()))) {
- String val = hMap.get(var.getName());
- if (val.startsWith("-const-")) {
- newValues.add(new ValueConstant(valMap.get(val)));
- } else {
- var.setName(val);
- newValues.add(var);
- }
- }
- } else {
- newValues.add(v);
- }
- }
-
- node.setArguments(newValues);
-
- }
-
-
- }
-
-
-
-
- public static class TupleVarRenamer extends QueryModelVisitorBase<RuntimeException> {
-
- private final HashMap<String, String> varChanges;
- private Map<String, Value> valMap;
-
- public TupleVarRenamer(HashMap<String, String> varChanges, Map<String, Value> valMap) {
- this.varChanges = varChanges;
- this.valMap = valMap;
- }
-
- @Override
- public void meet(ProjectionElemList node) {
- List<ProjectionElem> proj = node.getElements();
- for (ProjectionElem s : proj) {
- if (varChanges.containsKey(s.getSourceName())) {
- String name = s.getSourceName();
- s.setSourceName(varChanges.get(name));
- s.setTargetName(varChanges.get(name));
-
- }
- }
-
- }
-
-
- @Override
- public void meet(StatementPattern node) {
- SpVarReNamer spv = new SpVarReNamer(varChanges, valMap);
- node.visit(spv);
- }
-
-
- @Override
- public void meet(Filter node) {
- FilterVarReNamer fvr = new FilterVarReNamer(varChanges, valMap);
- node.getCondition().visit(fvr);
- node.getArg().visit(this);
-
- }
-
-
-
- }
-
- public static class VarCollector extends QueryModelVisitorBase<RuntimeException> {
-
- public static List<String> process(QueryModelNode node) {
- VarCollector collector = new VarCollector();
- node.visit(collector);
- return collector.getVarNames();
- }
-
- public static List<Var> processVar(QueryModelNode node) {
- VarCollector collector = new VarCollector();
- node.visit(collector);
- return collector.getVars();
- }
-
- private List<String> varNames = new ArrayList<String>();
- private List<Var> vars = Lists.newArrayList();
-
- public List<String> getVarNames() {
- return varNames;
- }
-
- public List<Var> getVars() {
- return vars;
- }
-
- @Override
- public void meet(Var var) {
- if (!var.hasValue()) {
- varNames.add(var.getName());
- }
- vars.add(var);
- }
- }
-
- public static class FilterVarValueCollector extends QueryModelVisitorBase<RuntimeException> {
-
- public static List<QueryModelNode> process(QueryModelNode node) {
- FilterVarValueCollector collector = new FilterVarValueCollector();
- node.visit(collector);
- return collector.getVars();
- }
-
-
-
- private List<QueryModelNode> vars = Lists.newArrayList();
-
-
- public List<QueryModelNode> getVars() {
- return vars;
- }
-
- @Override
- public void meet(Var node) {
- vars.add(node);
- }
-
- @Override
- public void meet(ValueConstant node) {
- vars.add(node);
- }
-
-
-
- }
-
-
-
-
- public static class NormalizeQueryVisitor extends QueryModelVisitorBase<Exception> {
-
- private TreeMap<String, List<QueryModelNode>> map = new TreeMap<String, List<QueryModelNode>>();
- private TreeMap<String, Integer> varMap = new TreeMap<String, Integer>();
- private TreeMap<String, Integer> emptyVarMap = new TreeMap<String, Integer>();
- private List<StatementPattern> statementList = new ArrayList<StatementPattern>();
- private List<QueryModelNode> filters = new ArrayList<QueryModelNode>();
- private boolean isIndex;
-
-
-
- public NormalizeQueryVisitor(boolean isIndex) {
- this.isIndex = isIndex;
- }
-
-
-
- private TreeMap<String, List<QueryModelNode>> getMap() {
-
- return map;
-
- }
-
-
- private TreeMap<String, Integer> getKeyMap() {
-
- return varMap;
- }
-
- private TreeMap<String, Integer> getVariableMap() {
- return emptyVarMap;
- }
-
- public List<StatementPattern> getStatementPatterns() {
- return statementList;
- }
-
-
- private List<QueryModelNode> getFilters() {
-
- return filters;
- }
-
- @Override
- public void meet(StatementPattern node) throws Exception {
-
- statementList.add(node);
-
- String s = "";
- String t = "";
-
- Var node1 = node.getSubjectVar();
- Var node2 = node.getObjectVar();
- Var node3 = node.getPredicateVar();
- Var node4 = node.getContextVar();
-
- String s1 = "";
- String s2 = "";
- String s3 = "";
- String s4 = "";
-
-
- if (node1.isConstant())
- s1 = node1.getName().substring(7);
-
- if (node2.isConstant())
- s2 = node2.getName().substring(7);
-
- if (node3.isConstant())
- s3 = node3.getName().substring(7);
-
- if (node4 != null) {
- if (node4.isConstant())
- s4 = node4.getName().substring(7);
- }
-
- if ((s1+s2+s3).length() == 0) {
- s = "Nonconstant nodes have no variables.";
- }
-
- List<QueryModelNode> nodes;
-
-
- if (s.length() > 0) {
-
- if (map.containsKey(s)) {
- nodes = map.get(s);
- nodes.add(node);
- } else {
- nodes = new ArrayList<QueryModelNode>();
- nodes.add(node);
- }
-
- map.put(s, nodes);
-
- } else {
-
- if (isIndex) {
-
- t = s1 + s2 + s3 + s4;
-
- if (map.containsKey(t)) {
- nodes = map.get(t);
- nodes.add(node);
- } else {
- nodes = new ArrayList<QueryModelNode>();
- nodes.add(node);
- }
-
- map.put(t, nodes);
-
- } else {
-
- String[] comps = new String[4];
- comps[0] = s1;
- comps[1] = s2;
- comps[2] = s3;
- comps[3] = s4;
-
- for (int i = 0; i < 3; i++) {
- if (comps[i].length() != 0) {
- if (map.containsKey(comps[i] + comps[3])) {
- nodes = map.get(comps[i] + comps[3]);
- nodes.add(node);
- } else {
- nodes = new ArrayList<QueryModelNode>();
- nodes.add(node);
- }
-
- map.put(comps[i] + comps[3], nodes);
-
- for (int j = i + 1; j < 3; j++) {
- if (comps[j].length() != 0) {
- if (map.containsKey(comps[i] + comps[j] + comps[3])) {
- nodes = map.get(comps[i] + comps[j] + comps[3]);
- nodes.add(node);
- } else {
- nodes = new ArrayList<QueryModelNode>();
- nodes.add(node);
- }
- map.put(comps[i] + comps[j] + comps[3], nodes);
- }
-
- }
- }
- }
-
- if (s1.length() != 0 && s2.length() != 0 && s3.length() != 0) {
- if (map.containsKey(s1 + s2 + s3 + s4)) {
- nodes = map.get(s1 + s2 + s3 + s4);
- nodes.add(node);
- } else {
- nodes = new ArrayList<QueryModelNode>();
- nodes.add(node);
- }
- map.put(s1 + s2 + s3 + s4, nodes);
- }
- }
- }
-
- super.meet(node);
-
- }
-
- @Override
- public void meet(Var node) throws Exception {
-
- int count = 1;
-
- if (!node.isConstant()) {
- if (varMap.containsKey(node.getName())) {
- count = varMap.get(node.getName());
- count++;
- varMap.put(node.getName(), count);
- } else
- varMap.put(node.getName(), 1);
-
- if (!emptyVarMap.containsKey(node.getName()))
- emptyVarMap.put(node.getName(), 0);
-
- }
- super.meet(node);
- }
-
- public void meet(Filter filter) throws Exception {
- filters.add(filter);
- super.meet(filter);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java
deleted file mode 100644
index dda452d..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java
+++ /dev/null
@@ -1,626 +0,0 @@
-package mvm.rya.indexing.external.tupleSet;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import info.aduna.iteration.CloseableIteration;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.NoSuchElementException;
-import java.util.Set;
-
-import mvm.rya.accumulo.precompQuery.AccumuloPrecompQueryIndexer;
-import mvm.rya.rdftriplestore.evaluation.ExternalBatchingIterator;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.io.Text;
-import org.openrdf.model.Literal;
-import org.openrdf.model.URI;
-import org.openrdf.model.impl.LiteralImpl;
-import org.openrdf.model.impl.URIImpl;
-import org.openrdf.query.Binding;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.MalformedQueryException;
-import org.openrdf.query.QueryEvaluationException;
-import org.openrdf.query.algebra.Projection;
-import org.openrdf.query.algebra.QueryModelNode;
-import org.openrdf.query.algebra.StatementPattern;
-import org.openrdf.query.algebra.ValueExpr;
-import org.openrdf.query.algebra.Var;
-import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
-import org.openrdf.query.impl.EmptyBindingSet;
-import org.openrdf.query.parser.ParsedTupleQuery;
-import org.openrdf.query.parser.sparql.SPARQLParser;
-import org.openrdf.repository.sail.SailRepositoryConnection;
-import org.openrdf.sail.SailException;
-
-import com.beust.jcommander.internal.Sets;
-import com.google.common.base.Joiner;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.HashBiMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-public class AccumuloIndexSet extends ExternalTupleSet implements ExternalBatchingIterator {
-
- private static final int WRITER_MAX_WRITE_THREADS = 30;
- private static final long WRITER_MAX_LATNECY = Long.MAX_VALUE;
- private static final long WRITER_MAX_MEMORY = 500L * 1024L * 1024L;
- private Map<String,AccValueFactory> bindings;
- private List<String> bindingslist;
- private final Connector accCon;
- private final String tablename;
- private long tableSize = 0;
- private List<String> varOrder = null;
-
-
- public static interface AccValueFactory {
- public org.openrdf.model.Value create(String str);
-
- public String create(org.openrdf.model.Value val);
- }
-
- public static class AccUrlFactory implements AccValueFactory {
- @Override
- public org.openrdf.model.Value create(final String str) {
- return new URIImpl(str);
- }
-
- @Override
- public String create(org.openrdf.model.Value val) {
- return val.stringValue();
- }
- }
-
- public static class AccValueFactoryImpl implements AccValueFactory {
- @Override
- public org.openrdf.model.Value create(final String str) {
- String[] split = str.split("\u0001");
- if (split.length > 1 && split[1].equals("1")) {
- return new URIImpl(split[0]);
- }
- if (split[0].contains(":")) {
- return new URIImpl(split[0]);
- }
- return new LiteralImpl(split[0]);
- }
-
- @Override
- public String create(org.openrdf.model.Value val) {
- if (val instanceof URI) {
- return val.stringValue() + "\u0001" + 1;
- }
- if (val instanceof Literal) {
- Literal v = (Literal) val;
- return v.getLabel() + "\u0001" + 2;
- }
- return null;
- }
- }
-
-
- //TODO set supportedVarOrderMap
- public AccumuloIndexSet(String sparql, SailRepositoryConnection conn, Connector accCon, String tablename) throws MalformedQueryException, SailException,
- QueryEvaluationException, MutationsRejectedException, TableNotFoundException {
- super(null);
- this.tablename = tablename;
- this.accCon = accCon;
- SPARQLParser sp = new SPARQLParser();
- ParsedTupleQuery pq = (ParsedTupleQuery) sp.parseQuery(sparql, null);
-
- setProjectionExpr((Projection) pq.getTupleExpr());
- CloseableIteration<BindingSet,QueryEvaluationException> iter = (CloseableIteration<BindingSet,QueryEvaluationException>) conn.getSailConnection()
- .evaluate(getTupleExpr(), null, new EmptyBindingSet(), false);
-
- BatchWriter w = accCon.createBatchWriter(tablename, WRITER_MAX_MEMORY, WRITER_MAX_LATNECY, WRITER_MAX_WRITE_THREADS);
- this.bindingslist = Lists.newArrayList(pq.getTupleExpr().getAssuredBindingNames());
-
- this.bindings = Maps.newHashMap();
-
- pq.getTupleExpr().visit(new QueryModelVisitorBase<RuntimeException>() {
- @Override
- public void meet(Var node) {
- QueryModelNode parent = node.getParentNode();
- if (parent instanceof StatementPattern) {
- StatementPattern statement = (StatementPattern) parent;
- if (node.equals(statement.getSubjectVar())) {
- bindings.put(node.getName(), new AccUrlFactory());
- }
- if (node.equals(statement.getPredicateVar())) {
- bindings.put(node.getName(), new AccUrlFactory());
- }
- if (node.equals(statement.getObjectVar())) {
- bindings.put(node.getName(), new AccValueFactoryImpl());
- }
- if (node.equals(statement.getContextVar())) {
- // TODO is this correct?
- bindings.put(node.getName(), new AccUrlFactory());
- }
- } else if(parent instanceof ValueExpr) {
- bindings.put(node.getName(), new AccValueFactoryImpl());
- }
- };
- });
-
-
-
-
-
- varOrder = new ArrayList<String>(bindingslist.size());
-
- while (iter.hasNext()) {
-
- BindingSet bs = iter.next();
- List<String> shiftBindingList = null;
- for (int j = 0; j < bindingslist.size(); j++) {
- StringBuffer sb = new StringBuffer();
- shiftBindingList = listShift(bindingslist, j); //TODO calling this each time not efficient
- String order = "";
- for (String b : shiftBindingList) {
- String val = bindings.get(b).create(bs.getValue(b));
- sb.append(val).append("\u0000");
- if (order.length() == 0) {
- order = b;
- } else {
- order = order + "\u0000" + b;
- }
- }
-
- if (varOrder.size() < bindingslist.size()) {
- varOrder.add(order);
- }
-
- //System.out.println("String buffer is " + sb);
- Mutation m = new Mutation(sb.deleteCharAt(sb.length() - 1).toString());
- m.put(new Text(varOrder.get(j)), new Text(""), new org.apache.accumulo.core.data.Value(new byte[]{}));
- w.addMutation(m);
- }
- tableSize += 1;
- }
-
- setLocalityGroups(tablename, accCon, varOrder);
- this.setSupportedVariableOrderMap(createSupportedVarOrderMap(varOrder));
-
-
- String orders = "";
-
- for(String s : varOrder) {
- s = s.replace("\u0000", ";");
- if(orders.length() == 0) {
- orders = s;
- } else {
- orders = orders + "\u0000" + s;
- }
- }
-
-
- Mutation m = new Mutation("~SPARQL");
- Value v = new Value(sparql.getBytes());
- m.put(new Text("" + tableSize), new Text(orders), v);
- w.addMutation(m);
-
- w.close();
- iter.close();
- }
-
-
-
-
- @Override
- public Map<String, Set<String>> getSupportedVariableOrders() {
-
- return this.getSupportedVariableOrderMap();
-
- }
-
-
- @Override
- public boolean supportsBindingSet(Set<String> bindingNames) {
-
- Map<String, Set<String>> varOrderMap = this.getSupportedVariableOrders();
- Collection<Set<String>> values = varOrderMap.values();
- Set<String> bNames = Sets.newHashSet();
-
- for (String s : this.getTupleExpr().getAssuredBindingNames()) {
- if (bindingNames.contains(s)) {
- bNames.add(s);
- }
- }
-
- return values.contains(bNames);
- }
-
-
- private String getVarOrder(Set<String> variables) {
-
- Map<String, Set<String>> varOrderMap = this.getSupportedVariableOrders();
-
- Set<Map.Entry<String, Set<String>>> entries = varOrderMap.entrySet();
-
- for (Map.Entry<String, Set<String>> e : entries) {
-
- if (e.getValue().equals(variables)) {
- return e.getKey();
- }
-
- }
-
- return null;
-
- }
-
- private String prefixToOrder(String order) {
-
- Map<String, String> invMap = HashBiMap.create(this.getTableVarMap()).inverse();
- String[] temp = order.split("\u0000");
-
- for (int i = 0; i < temp.length; i++) {
- temp[i] = this.getTableVarMap().get(temp[i]);
- }
-
- order = Joiner.on("\u0000").join(temp);
-
- for (String s : varOrder) {
- if (s.startsWith(order)) {
-
- temp = s.split("\u0000");
-
- for (int i = 0; i < temp.length; i++) {
- temp[i] = invMap.get(temp[i]);
- }
- return Joiner.on("\u0000").join(temp);
- }
- }
- throw new NoSuchElementException("Order is not a prefix of any locality group value!");
- }
-
- private String orderToLocGroup(List<String> order) {
- String localityGroup = "";
- for (String s : order) {
- if (localityGroup.length() == 0) {
- localityGroup = this.getTableVarMap().get(s);
- } else {
- localityGroup = localityGroup + "\u0000" + this.getTableVarMap().get(s);
- }
- }
- return localityGroup;
-
- }
-
-
- private void setLocalityGroups(String tableName, Connector conn, List<String> groups) {
-
- HashMap<String, Set<Text>> localityGroups = new HashMap<String, Set<Text>>();
-
-
-
- for (int i = 0; i < groups.size(); i++) {
- HashSet<Text> tempColumn = new HashSet<Text>();
- tempColumn.add(new Text(groups.get(i)));
- String groupName = groups.get(i).replace("\u0000","");
- localityGroups.put(groupName, tempColumn);
- }
-
-
- try {
- conn.tableOperations().setLocalityGroups(tableName, localityGroups);
- } catch (AccumuloException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (AccumuloSecurityException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (TableNotFoundException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
-
-
- }
-
-
-
-
-
-
-
- private List<String> listShift(List<String> list, int j) {
-
- if(j >= list.size()) {
- throw new IllegalArgumentException();
- }
-
- List<String> shiftList = Lists.newArrayList();
- for(int i=0; i<list.size(); i++) {
- shiftList.add(list.get((i+j)%list.size()));
- }
-
- return shiftList;
- }
-
-
-
- private Set<String> getConstantConstraints() {
-
- Map<String, String> tableMap = this.getTableVarMap();
- Set<String> keys = tableMap.keySet();
- Set<String> constants = Sets.newHashSet();
-
- for (String s : keys) {
- if (s.startsWith("-const-")) {
- constants.add(s);
- }
-
- }
-
- return constants;
-
- }
-
-
-
-
- public AccumuloIndexSet(String sparql, Connector accCon, String tablename) throws MalformedQueryException, SailException, QueryEvaluationException,
- MutationsRejectedException, TableNotFoundException {
- super(null);
- this.tablename = tablename;
- this.accCon = accCon;
- SPARQLParser sp = new SPARQLParser();
- ParsedTupleQuery pq = (ParsedTupleQuery) sp.parseQuery(sparql, null);
-
- setProjectionExpr((Projection) pq.getTupleExpr());
-
- this.bindingslist = Lists.newArrayList(pq.getTupleExpr().getAssuredBindingNames());
-
- this.bindings = Maps.newHashMap();
- pq.getTupleExpr().visit(new QueryModelVisitorBase<RuntimeException>() {
- @Override
- public void meet(Var node) {
- QueryModelNode parent = node.getParentNode();
- if (parent instanceof StatementPattern) {
- StatementPattern statement = (StatementPattern) parent;
- if (node.equals(statement.getSubjectVar())) {
- bindings.put(node.getName(), new AccUrlFactory());
- }
- if (node.equals(statement.getPredicateVar())) {
- bindings.put(node.getName(), new AccUrlFactory());
- }
- if (node.equals(statement.getObjectVar())) {
- bindings.put(node.getName(), new AccValueFactoryImpl());
- }
- if (node.equals(statement.getContextVar())) {
- // TODO is this correct?
- bindings.put(node.getName(), new AccUrlFactory());
- }
- } else if(parent instanceof ValueExpr) {
- bindings.put(node.getName(), new AccValueFactoryImpl());
- }
- };
- });
-
-
-
-
- Scanner s = accCon.createScanner(tablename, new Authorizations());
- s.setRange(Range.exact(new Text("~SPARQL")));
- Iterator<Entry<Key,Value>> i = s.iterator();
-
- String[] tempVarOrders = null;
-
- if (i.hasNext()) {
- Entry<Key, Value> entry = i.next();
- Text ts = entry.getKey().getColumnFamily();
- tempVarOrders = entry.getKey().getColumnQualifier().toString().split("\u0000");
- tableSize = Long.parseLong(ts.toString());
-
- } else {
- throw new IllegalStateException("Index table contains no metadata!");
- }
-
-
- varOrder = Lists.newArrayList();
-
- for(String t: tempVarOrders) {
- t = t.replace(";","\u0000");
- varOrder.add(t);
- }
-
- setLocalityGroups(tablename, accCon, varOrder);
- this.setSupportedVariableOrderMap(createSupportedVarOrderMap(varOrder));
-
- }
-
-
-
-
- private Map<String, Set<String>> createSupportedVarOrderMap(List<String> orders) {
-
- Map<String, Set<String>> supportedVars = Maps.newHashMap();
-
- for (String t : orders) {
-
- String[] tempOrder = t.split("\u0000");
- Set<String> varSet = Sets.newHashSet();
- String u = "";
-
- for (String s : tempOrder) {
- if(u.length() == 0) {
- u = s;
- } else{
- u = u+ "\u0000" + s;
- }
- varSet.add(s);
- supportedVars.put(u, new HashSet<String>(varSet));
-
- }
-
- }
-
- return supportedVars;
- }
-
-
-
- @Override
- public void setProjectionExpr(Projection tupleExpr) {
- super.setProjectionExpr(tupleExpr);
- this.bindingslist = Lists.newArrayList(tupleExpr.getAssuredBindingNames());
-
- this.bindings = Maps.newHashMap();
- tupleExpr.visit(new QueryModelVisitorBase<RuntimeException>() {
- @Override
- public void meet(Var node) {
- QueryModelNode parent = node.getParentNode();
- if (parent instanceof StatementPattern) {
- StatementPattern statement = (StatementPattern) parent;
- if (node.equals(statement.getSubjectVar())) {
- bindings.put(node.getName(), new AccUrlFactory());
- }
- if (node.equals(statement.getPredicateVar())) {
- bindings.put(node.getName(), new AccUrlFactory());
- }
- if (node.equals(statement.getObjectVar())) {
- bindings.put(node.getName(), new AccValueFactoryImpl());
- }
- if (node.equals(statement.getContextVar())) {
- // TODO is this correct?
- bindings.put(node.getName(), new AccUrlFactory());
- }
- } else if (parent instanceof ValueExpr) { //Add bindings associated with Filters
- bindings.put(node.getName(), new AccValueFactoryImpl());
- }
- };
- });
-
- }
-
- @Override
- public String getSignature() {
- return "AccumuloIndexSet(" + tablename + ") : " + Joiner.on(", ").join(bindingslist);
- }
-
- @Override
- public CloseableIteration<BindingSet,QueryEvaluationException> evaluate(BindingSet bindingset) throws QueryEvaluationException {
- return this.evaluate(Collections.singleton(bindingset));
- }
-
- @Override
- public double cardinality() {
- return tableSize;
- }
-
- @Override
- public CloseableIteration<BindingSet,QueryEvaluationException> evaluate(final Collection<BindingSet> bindingset) throws QueryEvaluationException {
-
- String localityGroup = "";
- Set<String> commonVars = Sets.newHashSet();
-
- if (!bindingset.isEmpty()) {
-
- BindingSet bs = bindingset.iterator().next();
- for (String b : bindingslist) {
- Binding v = bs.getBinding(b);
- if (v != null) {
- commonVars.add(b);
- }
-
- }
- }
-
- commonVars.addAll(getConstantConstraints());
- AccumuloPrecompQueryIndexer apq = null;
- List<String> fullVarOrder = null;
- try {
-
- if (commonVars.size() > 0) {
- String commonVarOrder = getVarOrder(commonVars);
- if(commonVarOrder == null) {
- throw new IllegalStateException("Index does not support binding set!");
- }
- fullVarOrder = Lists.newArrayList(prefixToOrder(commonVarOrder).split("\u0000"));
- localityGroup = orderToLocGroup(fullVarOrder);
- fullVarOrder.add("" + commonVars.size());
-
- } else {
- fullVarOrder = bindingslist;
- localityGroup = orderToLocGroup(fullVarOrder);
- fullVarOrder.add("" + 0);
- }
-
-
- apq = new AccumuloPrecompQueryIndexer(accCon, tablename);
- ValueMapVisitor vmv = new ValueMapVisitor();
- this.getTupleExpr().visit(vmv);
-
- return apq.queryPrecompJoin(fullVarOrder, localityGroup, this.bindings, vmv.getValMap(), bindingset);
-
- } catch(TableNotFoundException e) {
- throw new QueryEvaluationException(e);
- } finally {
- IOUtils.closeQuietly(apq);
- }
- }
-
-
- public class ValueMapVisitor extends QueryModelVisitorBase<RuntimeException> {
-
- Map<String, org.openrdf.model.Value> valMap = Maps.newHashMap();
-
-
- public Map<String, org.openrdf.model.Value> getValMap() {
- return valMap;
- }
-
- @Override
- public void meet(Var node) {
- if (node.getName().startsWith("-const-")) {
- valMap.put(node.getName(), node.getValue());
- }
-
- }
-
- }
-
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/ExternalTupleSet.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/ExternalTupleSet.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/ExternalTupleSet.java
deleted file mode 100644
index 0e2096d..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/ExternalTupleSet.java
+++ /dev/null
@@ -1,213 +0,0 @@
-package mvm.rya.indexing.external.tupleSet;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import info.aduna.iteration.CloseableIteration;
-
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.QueryEvaluationException;
-import org.openrdf.query.algebra.Projection;
-import org.openrdf.query.algebra.Var;
-import org.openrdf.query.algebra.evaluation.impl.ExternalSet;
-import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
-
-import com.beust.jcommander.internal.Sets;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Maps;
-
-/**
- * Abstract class for an External Tuple Set. This Tuple
- */
-public abstract class ExternalTupleSet extends ExternalSet {
-
- private Projection tupleExpr;
- private Map<String, String> tableVarMap = Maps.newHashMap();
- private Map<String, Set<String>> supportedVarOrders = Maps.newHashMap();
-
-
- public ExternalTupleSet() {
-
- }
-
- public ExternalTupleSet(Projection tupleExpr) {
- this.tupleExpr = tupleExpr;
- }
-
- @Override
- abstract public CloseableIteration<BindingSet, QueryEvaluationException> evaluate(BindingSet bindings) throws QueryEvaluationException;
-
- @Override
- public Set<String> getBindingNames() {
- return tupleExpr.getBindingNames();
- }
-
- @Override
- public Set<String> getAssuredBindingNames() {
- return tupleExpr.getAssuredBindingNames();
- }
-
- @Override
- public String getSignature() {
- return "(External Projection) " + Joiner.on(", ").join(tupleExpr.getProjectionElemList().getElements()).replaceAll("\\s+", " ");
- }
-
- public Projection getTupleExpr() {
- return tupleExpr;
- }
-
- public void setProjectionExpr(Projection tupleExpr) {
- this.tupleExpr = tupleExpr;
- }
-
-
- public void setTableVarMap(Map<String,String> vars) {
- this.tableVarMap = vars;
- }
-
-
- public Map<String, String> getTableVarMap() {
- return this.tableVarMap;
- }
-
-
- public void setSupportedVariableOrderMap(Map<String, Set<String>> varOrders) {
- this.supportedVarOrders = varOrders;
- }
-
-
- public Map<String, Set<String>> getSupportedVariableOrderMap() {
- return supportedVarOrders;
- }
-
-
- public void updateTupleExp(final Map<Var, Var> oldToNewBindings) {
- tupleExpr.visit(new QueryModelVisitorBase<RuntimeException>() {
- @Override
- public void meet(Var var) {
- if (oldToNewBindings.containsKey(var)) {
- var.replaceWith(oldToNewBindings.get(var));
- }
- }
- });
- }
-
- @Override
- public ExternalSet clone() {
- ExternalTupleSet clone = (ExternalTupleSet) super.clone();
- clone.tupleExpr = this.tupleExpr.clone();
- clone.tableVarMap = Maps.newHashMap();
- for(String s: this.tableVarMap.keySet()) {
- clone.tableVarMap.put(s,this.tableVarMap.get(s));
- }
- clone.supportedVarOrders = Maps.newHashMap();
- for(String s: this.supportedVarOrders.keySet()) {
- clone.supportedVarOrders.put(s,this.supportedVarOrders.get(s));
- }
- return clone;
- }
-
-
- public Map<String, Set<String>> getSupportedVariableOrders() {
-
- if (supportedVarOrders.size() != 0) {
- return supportedVarOrders;
- } else {
-
- Set<String> varSet = Sets.newHashSet();
- String t = "";
-
- for (String s : tupleExpr.getAssuredBindingNames()) {
- if (t.length() == 0) {
- t = s;
- } else {
- t = t + "\u0000" + s;
- }
-
- varSet.add(s);
- supportedVarOrders.put(t, new HashSet<String>(varSet));
-
- }
-
- return supportedVarOrders;
- }
- }
-
-
-
-
- public boolean supportsBindingSet(Set<String> bindingNames) {
-
- Map<String, Set<String>> varOrderMap = getSupportedVariableOrders();
- String bNames = "";
-
- for (String s : tupleExpr.getAssuredBindingNames()) {
- if (bindingNames.contains(s)) {
- if(bNames.length() == 0) {
- bNames = s;
- } else {
- bNames = bNames + "\u0000"+ s;
- }
- }
- }
-
- return varOrderMap.containsKey(bNames);
- }
-
-
-
- @Override
- public boolean equals(Object other) {
-
- if (!(other instanceof ExternalTupleSet)) {
- return false;
- } else {
-
- ExternalTupleSet arg = (ExternalTupleSet) other;
- if (this.getTupleExpr().equals(arg.getTupleExpr())) {
- return true;
- } else {
- return false;
- }
-
- }
-
- }
-
-
- @Override
- public int hashCode() {
- int result = 17;
- result = 31*result + tupleExpr.hashCode();
-
- return result;
- }
-
-
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/SimpleExternalTupleSet.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/SimpleExternalTupleSet.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/SimpleExternalTupleSet.java
deleted file mode 100644
index 44925ca..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/SimpleExternalTupleSet.java
+++ /dev/null
@@ -1,88 +0,0 @@
-package mvm.rya.indexing.external.tupleSet;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import info.aduna.iteration.CloseableIteration;
-
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.QueryEvaluationException;
-import org.openrdf.query.algebra.Projection;
-import org.openrdf.query.algebra.QueryModelVisitor;
-
-import com.google.common.base.Joiner;
-
-
-
-
-
-
-public class SimpleExternalTupleSet extends ExternalTupleSet {
-
-
-
- public SimpleExternalTupleSet(Projection tuple) {
- super();
- this.setProjectionExpr(tuple);
-
- }
-
- @Override
- public <X extends Exception> void visit(QueryModelVisitor<X> visitor)
- throws X
- {
- visitor.meetOther(this);
- }
-
- @Override
- public CloseableIteration<BindingSet, QueryEvaluationException> evaluate(BindingSet bindings)
- throws QueryEvaluationException {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public String getSignature() {
- return "(SimpleExternalTupleSet) "
- + Joiner.on(", ").join(this.getTupleExpr().getProjectionElemList().getElements()).replaceAll("\\s+", " ");
-
- }
-
- @Override
- public boolean equals(Object other) {
-
- if (!(other instanceof SimpleExternalTupleSet)) {
- return false;
- } else {
-
- SimpleExternalTupleSet arg = (SimpleExternalTupleSet) other;
- if (this.getTupleExpr().equals(arg.getTupleExpr())) {
- return true;
- } else {
- return false;
- }
-
- }
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java
deleted file mode 100644
index 4a708ab..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/AbstractMongoIndexer.java
+++ /dev/null
@@ -1,73 +0,0 @@
-package mvm.rya.indexing.mongodb;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import java.io.IOException;
-import java.util.Collection;
-
-import mvm.rya.api.domain.RyaStatement;
-import mvm.rya.api.domain.RyaURI;
-import mvm.rya.api.persist.index.RyaSecondaryIndexer;
-
-import org.apache.hadoop.conf.Configuration;
-
-public abstract class AbstractMongoIndexer implements RyaSecondaryIndexer {
-
- @Override
- public void close() throws IOException {
- }
-
- @Override
- public void flush() throws IOException {
- }
-
-
- @Override
- public Configuration getConf() {
- return null;
- }
-
-
- @Override
- public String getTableName() {
- return null;
- }
-
- @Override
- public void storeStatements(Collection<RyaStatement> ryaStatements)
- throws IOException {
- for (RyaStatement ryaStatement : ryaStatements){
- storeStatement(ryaStatement);
- }
-
- }
-
- @Override
- public void deleteStatement(RyaStatement stmt) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void dropGraph(RyaURI... graphs) {
- throw new UnsupportedOperationException();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/GeoMongoDBStorageStrategy.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/GeoMongoDBStorageStrategy.java b/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/GeoMongoDBStorageStrategy.java
deleted file mode 100644
index 0355225..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/mongodb/GeoMongoDBStorageStrategy.java
+++ /dev/null
@@ -1,185 +0,0 @@
-package mvm.rya.indexing.mongodb;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import mvm.rya.indexing.StatementContraints;
-import mvm.rya.indexing.accumulo.StatementSerializer;
-import mvm.rya.indexing.accumulo.geo.GeoParseUtils;
-
-import org.apache.commons.codec.binary.Hex;
-import org.openrdf.model.Statement;
-import org.openrdf.model.URI;
-
-import com.mongodb.BasicDBList;
-import com.mongodb.BasicDBObject;
-import com.mongodb.DBCollection;
-import com.mongodb.DBObject;
-import com.vividsolutions.jts.geom.Coordinate;
-import com.vividsolutions.jts.geom.Geometry;
-import com.vividsolutions.jts.io.ParseException;
-import com.vividsolutions.jts.io.WKTReader;
-
-public class GeoMongoDBStorageStrategy {
-
- private static final String ID = "_id";
- private static final String GEO = "location";
- private static final String CONTEXT = "context";
- private static final String PREDICATE = "predicate";
- private static final String OBJECT = "object";
- private static final String SUBJECT = "subject";
- public enum GeoQueryType {
- INTERSECTS {
- public String getKeyword() {
- return "$geoIntersects";
- }
- }, WITHIN {
- public String getKeyword() {
- return "$geoWithin";
- }
- },
- EQUALS {
- public String getKeyword() {
- return "$near";
- }
- };
-
- public abstract String getKeyword();
- }
-
- private double maxDistance;
-
-
- public GeoMongoDBStorageStrategy(double maxDistance) {
- this.maxDistance = maxDistance;
- }
-
- public void createIndices(DBCollection coll){
- coll.createIndex("{" + GEO + " : \"2dsphere\"" );
- }
-
- public DBObject getQuery(StatementContraints contraints, Geometry geo, GeoQueryType queryType) {
- BasicDBObject query;
- if (queryType.equals(GeoQueryType.EQUALS)){
- List<double[]> points = getCorrespondingPoints(geo);
- if (points.size() == 1){
- List circle = new ArrayList();
- circle.add(points.get(0));
- circle.add(maxDistance);
- BasicDBObject polygon = new BasicDBObject("$centerSphere", circle);
- query = new BasicDBObject(GEO, new BasicDBObject(GeoQueryType.WITHIN.getKeyword(), polygon));
- }else {
- query = new BasicDBObject(GEO, points);
- }
-
- }
- else {
- query = new BasicDBObject(GEO, new BasicDBObject(queryType.getKeyword(), new BasicDBObject("$polygon", getCorrespondingPoints(geo))));
- }
- if (contraints.hasSubject()){
- query.append(SUBJECT, contraints.getSubject().toString());
- }
- if (contraints.hasPredicates()){
- Set<URI> predicates = contraints.getPredicates();
- if (predicates.size() > 1){
- BasicDBList or = new BasicDBList();
- for (URI pred : predicates){
- DBObject currentPred = new BasicDBObject(PREDICATE, pred.toString());
- or.add(currentPred);
- }
- query.append("$or", or);
- }
- else if (!predicates.isEmpty()){
- query.append(PREDICATE, predicates.iterator().next().toString());
- }
- }
- if (contraints.hasContext()){
- query.append(CONTEXT, contraints.getContext().toString());
- }
-
- return query;
- }
-
-
- public Statement deserializeDBObject(DBObject queryResult) {
- Map result = queryResult.toMap();
- String subject = (String) result.get(SUBJECT);
- String object = (String) result.get(OBJECT);
- String predicate = (String) result.get(PREDICATE);
- String context = (String) result.get(CONTEXT);
- if (!context.isEmpty()){
- return StatementSerializer.readStatement(subject, predicate, object, context);
- }
- return StatementSerializer.readStatement(subject, predicate, object);
- }
-
-
-
- public DBObject serialize(Statement statement) throws ParseException{
- // if the object is wkt, then try to index it
- // write the statement data to the fields
- Geometry geo = (new WKTReader()).read(GeoParseUtils.getWellKnownText(statement));
- if(geo == null || geo.isEmpty() || !geo.isValid()) {
- throw new ParseException("Could not create geometry for statement " + statement);
- }
-
- String context = "";
- if (statement.getContext() != null){
- context = StatementSerializer.writeContext(statement);
- }
- String id = StatementSerializer.writeSubject(statement) + " " +
- StatementSerializer.writePredicate(statement) + " " + StatementSerializer.writeObject(statement) + " " + context;
- byte[] bytes = id.getBytes();
- try {
- MessageDigest digest = MessageDigest.getInstance("SHA-1");
- bytes = digest.digest(bytes);
- } catch (NoSuchAlgorithmException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- BasicDBObject doc = new BasicDBObject(ID, new String(Hex.encodeHex(bytes)))
- .append(GEO, getCorrespondingPoints(geo))
- .append(SUBJECT, StatementSerializer.writeSubject(statement))
- .append(PREDICATE, StatementSerializer.writePredicate(statement))
- .append(OBJECT, StatementSerializer.writeObject(statement))
- .append(CONTEXT, context);
- return doc;
-
- }
-
- private List<double[]> getCorrespondingPoints(Geometry geo){
- List<double[]> points = new ArrayList<double[]>();
- for (Coordinate coord : geo.getCoordinates()){
- points.add(new double[] {
- coord.x, coord.y
- });
- }
- return points;
-
- }
-
-}