You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rya.apache.org by pu...@apache.org on 2016/05/13 16:42:47 UTC
[5/6] incubator-rya git commit: RYA-64 - Integrated Rya PCJ Secondary
Index support into core Rya.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/14073a23/extras/indexing/src/main/java/mvm/rya/indexing/external/fluo/FluoPcjUpdaterSupplier.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/fluo/FluoPcjUpdaterSupplier.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/fluo/FluoPcjUpdaterSupplier.java
new file mode 100644
index 0000000..8d7b180
--- /dev/null
+++ b/extras/indexing/src/main/java/mvm/rya/indexing/external/fluo/FluoPcjUpdaterSupplier.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package mvm.rya.indexing.external.fluo;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import javax.annotation.ParametersAreNonnullByDefault;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Supplier;
+
+import io.fluo.api.client.FluoClient;
+import io.fluo.api.client.FluoFactory;
+import io.fluo.api.config.FluoConfiguration;
+import mvm.rya.indexing.external.PrecomputedJoinIndexerConfig;
+import mvm.rya.indexing.external.PrecomputedJoinIndexerConfig.PrecomputedJoinUpdaterType;
+
+/**
+ * Creates instances of {@link FluoPcjUpdater} using the values found in a {@link Configuration}.
+ */
+@ParametersAreNonnullByDefault
+public class FluoPcjUpdaterSupplier implements Supplier<FluoPcjUpdater> {
+
+ private final Supplier<Configuration> configSupplier;
+
+ /**
+ * Constructs an instance of {@link FluoPcjUpdaterSupplier}.
+ *
+ * @param configSupplier - Configures the {@link FluoPcjUpdater} that is supplied by this class. (not null)
+ */
+ public FluoPcjUpdaterSupplier(final Supplier<Configuration> configSupplier) {
+ this.configSupplier = checkNotNull(configSupplier);
+ }
+
+ @Override
+ public FluoPcjUpdater get() {
+ final Configuration config = configSupplier.get();
+ checkNotNull(config, "Could not create a FluoPcjUpdater because the application's configuration has not been provided yet.");
+
+ // Ensure the correct updater type has been set.
+ final PrecomputedJoinIndexerConfig indexerConfig = new PrecomputedJoinIndexerConfig(config);
+
+ final Optional<PrecomputedJoinUpdaterType> updaterType = indexerConfig.getPcjUpdaterType();
+ checkArgument(updaterType.isPresent() && (updaterType.get() == PrecomputedJoinUpdaterType.FLUO),
+ "This supplier requires the '" + PrecomputedJoinIndexerConfig.PCJ_UPDATER_TYPE +
+ "' value be set to '" + PrecomputedJoinUpdaterType.FLUO + "'.");
+
+ final FluoPcjUpdaterConfig fluoUpdaterConfig = new FluoPcjUpdaterConfig( indexerConfig.getConfig() );
+
+ // Make sure the required values are present.
+ checkArgument(fluoUpdaterConfig.getFluoAppName().isPresent(), "Missing configuration: " + FluoPcjUpdaterConfig.FLUO_APP_NAME);
+ checkArgument(fluoUpdaterConfig.getFluoZookeepers().isPresent(), "Missing configuration: " + FluoPcjUpdaterConfig.ACCUMULO_ZOOKEEPERS);
+ checkArgument(fluoUpdaterConfig.getAccumuloZookeepers().isPresent(), "Missing configuration: " + FluoPcjUpdaterConfig.ACCUMULO_ZOOKEEPERS);
+ checkArgument(fluoUpdaterConfig.getAccumuloInstance().isPresent(), "Missing configuration: " + FluoPcjUpdaterConfig.ACCUMULO_INSTANCE);
+ checkArgument(fluoUpdaterConfig.getAccumuloUsername().isPresent(), "Missing configuration: " + FluoPcjUpdaterConfig.ACCUMULO_USERNAME);
+ checkArgument(fluoUpdaterConfig.getAccumuloPassword().isPresent(), "Missing configuration: " + FluoPcjUpdaterConfig.ACCUMULO_PASSWORD);
+ checkArgument(fluoUpdaterConfig.getStatementVisibility().isPresent(), "Missing configuration: " + FluoPcjUpdaterConfig.STATEMENT_VISIBILITY);
+
+ // Fluo configuration values.
+ final FluoConfiguration fluoClientConfig = new FluoConfiguration();
+ fluoClientConfig.setApplicationName( fluoUpdaterConfig.getFluoAppName().get() );
+ fluoClientConfig.setInstanceZookeepers( fluoUpdaterConfig.getFluoZookeepers().get() );
+
+ // Accumulo Fluo Table configuration values.
+ fluoClientConfig.setAccumuloZookeepers( fluoUpdaterConfig.getAccumuloZookeepers().get() );
+ fluoClientConfig.setAccumuloInstance( fluoUpdaterConfig.getAccumuloInstance().get() );
+ fluoClientConfig.setAccumuloUser( fluoUpdaterConfig.getAccumuloUsername().get() );
+ fluoClientConfig.setAccumuloPassword( fluoUpdaterConfig.getAccumuloPassword().get() );
+
+ final FluoClient fluoClient = FluoFactory.newClient(fluoClientConfig);
+ final String statementVisibilities = fluoUpdaterConfig.getStatementVisibility().get();
+ return new FluoPcjUpdater(fluoClient, statementVisibilities);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/14073a23/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java
index 6011a52..5396926 100644
--- a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java
+++ b/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloIndexSet.java
@@ -1,5 +1,3 @@
-package mvm.rya.indexing.external.tupleSet;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -18,8 +16,7 @@ package mvm.rya.indexing.external.tupleSet;
* specific language governing permissions and limitations
* under the License.
*/
-
-import info.aduna.iteration.CloseableIteration;
+package mvm.rya.indexing.external.tupleSet;
import java.util.ArrayList;
import java.util.Collection;
@@ -31,20 +28,16 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
-import mvm.rya.accumulo.precompQuery.AccumuloPcjQuery;
-import mvm.rya.api.utils.IteratorWrapper;
-import mvm.rya.indexing.PcjQuery;
-import mvm.rya.indexing.external.tupleSet.PcjTables.PcjException;
-import mvm.rya.indexing.external.tupleSet.PcjTables.PcjMetadata;
-import mvm.rya.indexing.external.tupleSet.PcjTables.VariableOrder;
-import mvm.rya.rdftriplestore.evaluation.ExternalBatchingIterator;
-
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.hadoop.io.Text;
+import org.apache.rya.indexing.pcj.storage.PcjException;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjTables;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
import org.openrdf.query.Binding;
import org.openrdf.query.BindingSet;
import org.openrdf.query.MalformedQueryException;
@@ -64,6 +57,12 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import info.aduna.iteration.CloseableIteration;
+import mvm.rya.accumulo.precompQuery.AccumuloPcjQuery;
+import mvm.rya.api.utils.IteratorWrapper;
+import mvm.rya.indexing.PcjQuery;
+import mvm.rya.rdftriplestore.evaluation.ExternalBatchingIterator;
+
/**
* During query planning, this node is inserted into the parsed query to
* represent part of the original query (a sub-query). This sub-query is the
@@ -95,7 +94,7 @@ public class AccumuloIndexSet extends ExternalTupleSet implements ExternalBatchi
private final Connector accCon; //connector to Accumulo table where results are stored
private final String tablename; //name of Accumulo table
private List<String> varOrder = null; // orders in which results are written to table
- private PcjTables pcj = new PcjTables();
+ private final PcjTables pcj = new PcjTables();
@Override
public Map<String, Set<String>> getSupportedVariableOrders() {
@@ -118,14 +117,14 @@ public class AccumuloIndexSet extends ExternalTupleSet implements ExternalBatchi
* @throws MutationsRejectedException
* @throws TableNotFoundException
*/
- public AccumuloIndexSet(String sparql, Connector accCon, String tablename) throws MalformedQueryException, SailException, QueryEvaluationException,
+ public AccumuloIndexSet(final String sparql, final Connector accCon, final String tablename) throws MalformedQueryException, SailException, QueryEvaluationException,
MutationsRejectedException, TableNotFoundException {
this.tablename = tablename;
this.accCon = accCon;
final SPARQLParser sp = new SPARQLParser();
final ParsedTupleQuery pq = (ParsedTupleQuery) sp.parseQuery(sparql, null);
- Optional<Projection> projection = new ParsedQueryUtil().findProjection(pq);
+ final Optional<Projection> projection = new ParsedQueryUtil().findProjection(pq);
if(!projection.isPresent()) {
throw new MalformedQueryException("SPARQL query '" + sparql + "' does not contain a Projection.");
}
@@ -156,7 +155,7 @@ public class AccumuloIndexSet extends ExternalTupleSet implements ExternalBatchi
* @throws MutationsRejectedException
* @throws TableNotFoundException
*/
- public AccumuloIndexSet(Connector accCon, String tablename)
+ public AccumuloIndexSet(final Connector accCon, final String tablename)
throws MalformedQueryException, SailException,
QueryEvaluationException, MutationsRejectedException,
TableNotFoundException {
@@ -191,7 +190,7 @@ public class AccumuloIndexSet extends ExternalTupleSet implements ExternalBatchi
double cardinality = 0;
try {
cardinality = pcj.getPcjMetadata(accCon, tablename).getCardinality();
- } catch (PcjException e) {
+ } catch (final PcjException e) {
e.printStackTrace();
}
return cardinality;
@@ -207,7 +206,7 @@ public class AccumuloIndexSet extends ExternalTupleSet implements ExternalBatchi
* Sets locality groups for more efficient scans - these are usually the variable
* orders in the table so that scans for specific orders are more efficient
*/
- private void setLocalityGroups(String tableName, Connector conn, List<String> groups) {
+ private void setLocalityGroups(final String tableName, final Connector conn, final List<String> groups) {
final HashMap<String, Set<Text>> localityGroups = new HashMap<String, Set<Text>>();
for (int i = 0; i < groups.size(); i++) {
@@ -229,7 +228,7 @@ public class AccumuloIndexSet extends ExternalTupleSet implements ExternalBatchi
@Override
- public CloseableIteration<BindingSet,QueryEvaluationException> evaluate(BindingSet bindingset) throws QueryEvaluationException {
+ public CloseableIteration<BindingSet,QueryEvaluationException> evaluate(final BindingSet bindingset) throws QueryEvaluationException {
return this.evaluate(Collections.singleton(bindingset));
}
@@ -298,7 +297,7 @@ public class AccumuloIndexSet extends ExternalTupleSet implements ExternalBatchi
* is just the variable order expressed in terms of the variables stored
* in the table
*/
- private String orderToLocGroup(List<String> order) {
+ private String orderToLocGroup(final List<String> order) {
String localityGroup = "";
for (final String s : order) {
if (localityGroup.length() == 0) {
@@ -347,7 +346,7 @@ public class AccumuloIndexSet extends ExternalTupleSet implements ExternalBatchi
* @return - string representation of the Set variables, in an order that is in the
* table
*/
- private String getVarOrder(Set<String> variables) {
+ private String getVarOrder(final Set<String> variables) {
final Map<String, Set<String>> varOrderMap = this.getSupportedVariableOrders();
final Set<Map.Entry<String, Set<String>>> entries = varOrderMap.entrySet();
for (final Map.Entry<String, Set<String>> e : entries) {
@@ -386,7 +385,7 @@ public class AccumuloIndexSet extends ExternalTupleSet implements ExternalBatchi
return valMap;
}
@Override
- public void meet(Var node) {
+ public void meet(final Var node) {
if (node.getName().startsWith("-const-")) {
valMap.put(node.getName(), node.getValue());
}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/14073a23/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloPcjSerializer.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloPcjSerializer.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloPcjSerializer.java
deleted file mode 100644
index bb87d7f..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/AccumuloPcjSerializer.java
+++ /dev/null
@@ -1,187 +0,0 @@
-package mvm.rya.indexing.external.tupleSet;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkArgument;
-import static mvm.rya.api.RdfCloudTripleStoreConstants.DELIM_BYTE;
-import static mvm.rya.api.RdfCloudTripleStoreConstants.DELIM_BYTES;
-import static mvm.rya.api.RdfCloudTripleStoreConstants.TYPE_DELIM_BYTE;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import javax.annotation.ParametersAreNonnullByDefault;
-
-import mvm.rya.api.domain.RyaType;
-import mvm.rya.api.resolver.RdfToRyaConversions;
-import mvm.rya.api.resolver.RyaContext;
-import mvm.rya.api.resolver.RyaToRdfConversions;
-import mvm.rya.api.resolver.RyaTypeResolverException;
-import mvm.rya.indexing.external.tupleSet.PcjTables.VariableOrder;
-
-import org.openrdf.model.Value;
-import org.openrdf.query.Binding;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.algebra.evaluation.QueryBindingSet;
-
-import com.google.common.primitives.Bytes;
-
-/**
- * Converts {@link BindingSet}s to byte[]s and back again. The bytes do not
- * include the binding names and are ordered with a {@link VariableOrder}.
- */
-@ParametersAreNonnullByDefault
-public class AccumuloPcjSerializer implements BindingSetConverter<byte[]> {
-
- @Override
- public byte[] convert(BindingSet bindingSet, VariableOrder varOrder) throws BindingSetConversionException {
- checkNotNull(bindingSet);
- checkNotNull(varOrder);
- checkBindingsSubsetOfVarOrder(bindingSet, varOrder);
-
- // A list that holds all of the byte segments that will be concatenated at the end.
- // This minimizes byte[] construction.
- final List<byte[]> byteSegments = new LinkedList<>();
-
- try {
- for(final String varName: varOrder) {
- // Only write information for a variable name if the binding set contains it.
- if(bindingSet.hasBinding(varName)) {
- final RyaType rt = RdfToRyaConversions.convertValue(bindingSet.getBinding(varName).getValue());
- final byte[][] serializedVal = RyaContext.getInstance().serializeType(rt);
- byteSegments.add(serializedVal[0]);
- byteSegments.add(serializedVal[1]);
- }
-
- // But always write the value delimiter. If a value is missing, you'll see two delimiters next to each-other.
- byteSegments.add(DELIM_BYTES);
- }
-
- return concat(byteSegments);
- } catch (RyaTypeResolverException e) {
- throw new BindingSetConversionException("Could not convert the BindingSet into a byte[].", e);
- }
- }
-
- @Override
- public BindingSet convert(byte[] bindingSetBytes, VariableOrder varOrder) throws BindingSetConversionException {
- checkNotNull(bindingSetBytes);
- checkNotNull(varOrder);
-
- try {
- // Slice the row into bindings.
- List<byte[]> values = splitlByDelimByte(bindingSetBytes);
- String[] varOrderStrings = varOrder.toArray();
- checkArgument(values.size() == varOrderStrings.length);
-
- // Convert the Binding bytes into a BindingSet.
- final QueryBindingSet bindingSet = new QueryBindingSet();
-
- for(int i = 0; i < varOrderStrings.length; i++) {
- byte[] valueBytes = values.get(i);
- if(valueBytes.length > 0) {
- String name = varOrderStrings[i];
- Value value = deserializeValue(valueBytes);
- bindingSet.addBinding(name, value);
- }
- }
-
- return bindingSet;
- } catch (RyaTypeResolverException e) {
- throw new BindingSetConversionException("Could not convert the byte[] into a BindingSet.", e);
- }
- }
-
- /**
- * Checks to see if the names of all the {@link Binding}s in the {@link BindingSet}
- * are a subset of the variables names in {@link VariableOrder}.
- *
- * @param bindingSet - The binding set whose Bindings will be inspected. (not null)
- * @param varOrder - The names of the bindings that may appear in the BindingSet. (not null)
- * @throws IllegalArgumentException Indicates the names of the bindings are
- * not a subset of the variable order.
- */
- private static void checkBindingsSubsetOfVarOrder(BindingSet bindingSet, VariableOrder varOrder) throws IllegalArgumentException {
- checkNotNull(bindingSet);
- checkNotNull(varOrder);
-
- Set<String> bindingNames = bindingSet.getBindingNames();
- List<String> varNames = varOrder.getVariableOrders();
- checkArgument(varNames.containsAll(bindingNames), "The BindingSet contains a Binding whose name is not part of the VariableOrder.");
- }
-
- private static final byte[] concat(Iterable<byte[]> byteSegments) {
- checkNotNull(byteSegments);
-
- // Allocate a byte array that is able to hold the segments.
- int length = 0;
- for(byte[] byteSegment : byteSegments) {
- length += byteSegment.length;
- }
- byte[] result = new byte[length];
-
- // Copy the segments to the byte array and return it.
- ByteBuffer buff = ByteBuffer.wrap(result);
- for(byte[] byteSegment : byteSegments) {
- buff.put(byteSegment);
- }
- return result;
- }
-
- private static List<byte[]> splitlByDelimByte(byte[] bindingSetBytes) {
- checkNotNull(bindingSetBytes);
-
- List<byte[]> values = new LinkedList<>();
-
- ByteBuffer buff = ByteBuffer.wrap(bindingSetBytes);
- int start = 0;
- while(buff.hasRemaining()) {
- if(buff.get() == DELIM_BYTE) {
- // Mark the position of the value delimiter.
- int end = buff.position();
-
- // Move to the start of the value and copy the bytes into an array.
- byte[] valueBytes = new byte[(end - start) -1];
- buff.position(start);
- buff.get(valueBytes);
- buff.position(end);
- values.add(valueBytes);
-
- // Move the start of the next value to the end of this one.
- start = end;
- }
- }
-
- return values;
- }
-
- private static Value deserializeValue(byte[] byteVal) throws RyaTypeResolverException {
- final int typeIndex = Bytes.indexOf(byteVal, TYPE_DELIM_BYTE);
- checkArgument(typeIndex >= 0);
- final byte[] data = Arrays.copyOf(byteVal, typeIndex);
- final byte[] type = Arrays.copyOfRange(byteVal, typeIndex, byteVal.length);
- final RyaType rt = RyaContext.getInstance().deserialize(Bytes.concat(data,type));
- return RyaToRdfConversions.convertValue(rt);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/14073a23/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/BindingSetConverter.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/BindingSetConverter.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/BindingSetConverter.java
deleted file mode 100644
index 84aee0c..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/BindingSetConverter.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package mvm.rya.indexing.external.tupleSet;
-
-import javax.annotation.ParametersAreNonnullByDefault;
-
-import org.openrdf.query.Binding;
-import org.openrdf.query.BindingSet;
-
-import mvm.rya.indexing.external.tupleSet.PcjTables.VariableOrder;
-
-/**
- * Converts {@link BindingSet}s into other representations. This library is
- * intended to convert between BindingSet and whatever format it is being
- * stored as. These formats are often optimized for query evaluation.
- *
- * @param <T> Defines the type of model {@link BindingSet}s will be converted into/from.
- */
-@ParametersAreNonnullByDefault
-public interface BindingSetConverter<T> {
-
- /**
- * Converts a {@link BindingSet} into the target model. The target model
- * may not include every {@link Binding} that was in the original BindingSet,
- * it may not include the binding names, and it may order the binding values.
- * All of this information is specified using a {@link VariableOrder}.
- * </p>
- * Because the resulting model may not include the binding names from the
- * original object, you must hold onto that information if you want to
- * convert the resulting model back into a BindingSet later. Because the
- * resulting model may only contain a subset of the original BindingSet's
- * bindings, some information may be lost, so you may not be able to convert
- * the target model back into the original BindingSet.
- *
- * @param bindingSet - The BindingSet that will be converted. (not null)
- * @param varOrder - Which bindings and in what order they will appear in the
- * resulting model. (not null)
- * @return The BindingSet formatted as the target model.
- * @throws BindingSetConversionException The BindingSet was unable to be
- * converted into the target model. This will happen if the BindingSet has
- * a binding whose name is not in the VariableOrder or if one of the values
- * could not be converted into the target model.
- */
- public T convert(BindingSet bindingSet, VariableOrder varOrder) throws BindingSetConversionException;
-
- /**
- * Converts the target model representation of a {@link BindingSet} as is
- * created by {@link #convert(BindingSet, VariableOrder)} back into a
- * BindingSet.
- * </p>
- * You must provide the Binding names and the order they were written to
- * by using a {@link VariableOrder}.
- * </p>
- * If there is no value for one of the variable order names, then that binding
- * will be missing from the resulting BindingSet.
- *
- * @param bindingSet - The BindingSet formatted as the target model that will
- * be converted. (not null)
- * @param varOrder - The VariableOrder that was used to create the target model. (not null)
- * @return The {@link BindingSet} representation of the target model.
- * @throws BindingSetConversionException The target model was unable to be
- * converted back into a BindingSet.
- */
- public BindingSet convert(T bindingSet, VariableOrder varOrder) throws BindingSetConversionException;
-
- /**
- * One of the conversion methods of {@link BindingSetConverter} was unable to
- * to convert the {@link BindingSet} to/from the converted model.
- */
- public static class BindingSetConversionException extends Exception {
- private static final long serialVersionUID = 1L;
-
- /**
- * Constructs an instance of {@link BindingSetConversionException}.
- *
- * @param message - Describes why this exception was thrown.
- */
- public BindingSetConversionException(final String message) {
- super(message);
- }
-
- /**
- * BindingSetConversionException
- *
- * @param message - Describes why this exception was thrown.
- * @param cause - The exception that caused this one to be thrown.
- */
- public BindingSetConversionException(final String message, final Throwable cause) {
- super(message, cause);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/14073a23/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/BindingSetStringConverter.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/BindingSetStringConverter.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/BindingSetStringConverter.java
deleted file mode 100644
index c4116f2..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/BindingSetStringConverter.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package mvm.rya.indexing.external.tupleSet;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-import javax.annotation.ParametersAreNonnullByDefault;
-
-import org.openrdf.model.URI;
-import org.openrdf.model.Value;
-import org.openrdf.model.ValueFactory;
-import org.openrdf.model.impl.URIImpl;
-import org.openrdf.model.impl.ValueFactoryImpl;
-import org.openrdf.model.vocabulary.XMLSchema;
-import org.openrdf.query.Binding;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.algebra.evaluation.QueryBindingSet;
-
-import com.google.common.base.Joiner;
-
-import mvm.rya.api.domain.RyaType;
-import mvm.rya.api.resolver.RdfToRyaConversions;
-import mvm.rya.indexing.external.tupleSet.PcjTables.VariableOrder;
-
-/**
- * Converts {@link BindingSet}s to Strings and back again. The Strings do not
- * include the binding names and are ordered with a {@link VariableOrder}.
- */
-@ParametersAreNonnullByDefault
-public class BindingSetStringConverter implements BindingSetConverter<String> {
-
- public static final String BINDING_DELIM = ":::";
- public static final String TYPE_DELIM = "<<~>>";
- public static final String NULL_VALUE_STRING = Character.toString( '\0' );
-
- private static final ValueFactory valueFactory = new ValueFactoryImpl();
-
- @Override
- public String convert(final BindingSet bindingSet, final VariableOrder varOrder) {
- checkBindingsSubsetOfVarOrder(bindingSet, varOrder);
-
- // Convert each Binding to a String.
- final List<String> bindingStrings = new ArrayList<>();
- for(final String varName : varOrder) {
- if(bindingSet.hasBinding(varName)) {
- // Add a value to the binding set.
- final Value value = bindingSet.getBinding(varName).getValue();
- final RyaType ryaValue = RdfToRyaConversions.convertValue(value);
- final String bindingString = ryaValue.getData() + TYPE_DELIM + ryaValue.getDataType();
- bindingStrings.add(bindingString);
- } else {
- // Add a null value to the binding set.
- bindingStrings.add(NULL_VALUE_STRING);
- }
- }
-
- // Join the bindings using the binding delim.
- return Joiner.on(BINDING_DELIM).join(bindingStrings);
- }
-
- /**
- * Checks to see if the names of all the {@link Binding}s in the {@link BindingSet}
- * are a subset of the variables names in {@link VariableOrder}.
- *
- * @param bindingSet - The binding set whose Bindings will be inspected. (not null)
- * @param varOrder - The names of the bindings that may appear in the BindingSet. (not null)
- * @throws IllegalArgumentException Indicates the names of the bindings are
- * not a subset of the variable order.
- */
- private static void checkBindingsSubsetOfVarOrder(final BindingSet bindingSet, final VariableOrder varOrder) throws IllegalArgumentException {
- checkNotNull(bindingSet);
- checkNotNull(varOrder);
-
- final Set<String> bindingNames = bindingSet.getBindingNames();
- final List<String> varNames = varOrder.getVariableOrders();
- checkArgument(varNames.containsAll(bindingNames), "The BindingSet contains a Binding whose name is not part of the VariableOrder.");
- }
-
- @Override
- public BindingSet convert(final String bindingSetString, final VariableOrder varOrder) {
- checkNotNull(bindingSetString);
- checkNotNull(varOrder);
-
- final String[] bindingStrings = bindingSetString.split(BINDING_DELIM);
- final String[] varOrrderArr = varOrder.toArray();
- checkArgument(varOrrderArr.length == bindingStrings.length, "The number of Bindings must match the length of the VariableOrder.");
-
- final QueryBindingSet bindingSet = new QueryBindingSet();
- for(int i = 0; i < bindingStrings.length; i++) {
- final String bindingString = bindingStrings[i];
- if(!NULL_VALUE_STRING.equals(bindingString)) {
- final String name = varOrrderArr[i];
- final Value value = toValue(bindingStrings[i]);
- bindingSet.addBinding(name, value);
- }
- }
- return bindingSet;
- }
-
- /**
- * Creates a {@link Value} from a String representation of it.
- *
- * @param valueString - The String representation of the value. (not null)
- * @return The {@link Value} representation of the String.
- */
- protected static Value toValue(final String valueString) {
- checkNotNull(valueString);
-
- // Split the String that was stored in Fluo into its Value and Type parts.
- final String[] valueAndType = valueString.split(TYPE_DELIM);
- if(valueAndType.length != 2) {
- throw new IllegalArgumentException("Array must contain data and type info!");
- }
-
- final String dataString = valueAndType[0];
- final String typeString = valueAndType[1];
-
- // Convert the String Type into a URI that describes the type.
- final URI typeURI = valueFactory.createURI(typeString);
-
- // Convert the String Value into a Value.
- final Value value = typeURI.equals(XMLSchema.ANYURI) ?
- valueFactory.createURI(dataString) :
- valueFactory.createLiteral(dataString, new URIImpl(typeString));
-
- return value;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/14073a23/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/PcjTables.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/PcjTables.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/PcjTables.java
deleted file mode 100644
index d8a81a6..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/PcjTables.java
+++ /dev/null
@@ -1,833 +0,0 @@
-package mvm.rya.indexing.external.tupleSet;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Objects;
-import java.util.Set;
-
-import javax.annotation.ParametersAreNonnullByDefault;
-import javax.annotation.concurrent.Immutable;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.ConditionalWriter;
-import org.apache.accumulo.core.client.ConditionalWriterConfig;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.MutationsRejectedException;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableExistsException;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.admin.TableOperations;
-import org.apache.accumulo.core.client.lexicoder.ListLexicoder;
-import org.apache.accumulo.core.client.lexicoder.LongLexicoder;
-import org.apache.accumulo.core.client.lexicoder.StringLexicoder;
-import org.apache.accumulo.core.data.Condition;
-import org.apache.accumulo.core.data.ConditionalMutation;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
-import org.openrdf.query.BindingSet;
-import org.openrdf.query.MalformedQueryException;
-import org.openrdf.query.QueryEvaluationException;
-import org.openrdf.query.QueryLanguage;
-import org.openrdf.query.TupleQuery;
-import org.openrdf.query.TupleQueryResult;
-import org.openrdf.repository.RepositoryConnection;
-import org.openrdf.repository.RepositoryException;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Lists;
-
-import mvm.rya.indexing.accumulo.VisibilityBindingSet;
-import mvm.rya.indexing.external.tupleSet.BindingSetConverter.BindingSetConversionException;
-
-/**
- * Functions that create and maintain the PCJ tables that are used by Rya.
- */
-@ParametersAreNonnullByDefault
-public class PcjTables {
- private static final Logger log = Logger.getLogger(PcjTables.class);
-
- /**
- * The Row ID of all {@link PcjMetadata} entries that are stored in Accumulo.
- */
- private static final Text PCJ_METADATA_ROW_ID = new Text("pcjMetadata");
-
- /**
- * The Column Family for all PCJ metadata entries.
- */
- private static final Text PCJ_METADATA_FAMILY = new Text("metadata");
-
- /**
- * The Column Qualifier for the SPARQL query a PCJ is built from.
- */
- private static final Text PCJ_METADATA_SPARQL_QUERY = new Text("sparql");
-
- /**
- * The Column Qualifier for the cardinality of a PCJ.
- */
- private static final Text PCJ_METADATA_CARDINALITY = new Text("cardinality");
-
- /**
- * The Column Qualifier for the various variable orders a PCJ's results are written to.
- */
- private static final Text PCJ_METADATA_VARIABLE_ORDERS = new Text("variableOrders");
-
- // Lexicoders used to read/write PcjMetadata to/from Accumulo.
- private static final LongLexicoder longLexicoder = new LongLexicoder();
- private static final StringLexicoder stringLexicoder = new StringLexicoder();
- private static final ListLexicoder<String> listLexicoder = new ListLexicoder<>(stringLexicoder);
-
- /**
- * An ordered list of {@link BindingSet} variable names. These are used to
- * specify the order {@link Binding}s within the set are serialized to Accumulo.
- * This order effects which rows a prefix scan will hit.
- */
- @Immutable
- @ParametersAreNonnullByDefault
- public static final class VariableOrder implements Iterable<String> {
-
- public static final String VAR_ORDER_DELIM = ";";
-
- private final ImmutableList<String> variableOrder;
-
- /**
- * Constructs an instance of {@link VariableOrder}.
- *
- * @param varOrder - An ordered array of Binding Set variables. (not null)
- */
- public VariableOrder(final String... varOrder) {
- checkNotNull(varOrder);
- variableOrder = ImmutableList.copyOf(varOrder);
- }
-
- /**
- * Constructs an instance of {@link VariableOrdeR{.
- *
- * @param varOrder - An ordered collection of Binding Set variables. (not null)
- */
- public VariableOrder(final Collection<String> varOrder) {
- checkNotNull(varOrder);
- variableOrder = ImmutableList.copyOf(varOrder);
- }
-
- /**
- * Constructs an instance of {@link VariableOrder}.
- *
- * @param varOrderString - The String representation of a VariableOrder. (not null)
- */
- public VariableOrder(final String varOrderString) {
- checkNotNull(varOrderString);
- variableOrder = ImmutableList.copyOf( varOrderString.split(VAR_ORDER_DELIM) );
- }
-
- /**
- * @return And ordered list of Binding Set variables.
- */
- public ImmutableList<String> getVariableOrders() {
- return variableOrder;
- }
-
- /**
- * @return The variable order as an ordered array of Strings. This array is mutable.
- */
- public String[] toArray() {
- final String[] array = new String[ variableOrder.size() ];
- return variableOrder.toArray( array );
- }
-
- @Override
- public String toString() {
- return Joiner.on(VAR_ORDER_DELIM).join(variableOrder);
- }
-
- @Override
- public int hashCode() {
- return variableOrder.hashCode();
- }
-
- @Override
- public boolean equals(final Object o) {
- if(this == o) {
- return true;
- } else if(o instanceof VariableOrder) {
- final VariableOrder varOrder = (VariableOrder) o;
- return variableOrder.equals( varOrder.variableOrder );
- }
- return false;
- }
-
- @Override
- public Iterator<String> iterator() {
- return variableOrder.iterator();
- }
- }
-
- /**
- * Metadata that is stored in a PCJ table about the results that are stored within it.
- */
- @Immutable
- @ParametersAreNonnullByDefault
- public static final class PcjMetadata {
- private final String sparql;
- private final long cardinality;
- private final ImmutableSet<VariableOrder> varOrders;
-
- /**
- * Constructs an instance of {@link PcjMetadata}.
- *
- * @param sparql - The SPARQL query this PCJ solves. (not null)
- * @param cardinality - The number of results the PCJ has. (>= 0)
- * @param varOrders - Strings that describe each of the variable orders
- * the results are stored in. (not null)
- */
- public PcjMetadata(final String sparql, final long cardinality, final Collection<VariableOrder> varOrders) {
- this.sparql = checkNotNull(sparql);
- this.varOrders = ImmutableSet.copyOf( checkNotNull(varOrders) );
-
- checkArgument(cardinality >= 0, "Cardinality of a PCJ must be >= 0. Was: " + cardinality);
- this.cardinality = cardinality;
- }
-
- /**
- * @return The SPARQL query this PCJ solves.
- */
- public String getSparql() {
- return sparql;
- }
-
- /**
- * @return The number of results the PCJ has.
- */
- public long getCardinality() {
- return cardinality;
- }
-
- /**
- * @return Strings that describe each of the variable orders the results are stored in.
- */
- public ImmutableSet<VariableOrder> getVarOrders() {
- return varOrders;
- }
-
- /**
- * Updates the cardinality of a {@link PcjMetadata} by a {@code delta}.
- *
- * @param metadata - The PCJ metadata to update. (not null)
- * @param delta - How much the cardinality of the PCJ has changed.
- * @return A new instance of {@link PcjMetadata} with the new cardinality.
- */
- public static PcjMetadata updateCardinality(final PcjMetadata metadata, final int delta) {
- checkNotNull(metadata);
- return new PcjMetadata(metadata.sparql, metadata.cardinality + delta, metadata.varOrders);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(sparql, cardinality, varOrders);
- }
-
- @Override
- public boolean equals(final Object o) {
- if(this == o) {
- return true;
- } else if(o instanceof PcjMetadata) {
- final PcjMetadata metadata = (PcjMetadata) o;
- return new EqualsBuilder()
- .append(sparql, metadata.sparql)
- .append(cardinality, metadata.cardinality)
- .append(varOrders, metadata.varOrders)
- .build();
- }
- return false;
- }
- }
-
- /**
- * Creates Accumulo table names that may be recognized by Rya as a table that
- * holds the results of a Precomputed Join.
- */
- public static class PcjTableNameFactory {
-
- /**
- * Creates an Accumulo table names that may be recognized by Rya as a table
- * that holds the results of a Precomputed Join.
- * </p>
- * An Accumulo cluster may host more than one Rya instance. To ensure each
- * Rya instance's RDF Triples are segregated from each other, they are stored
- * within different Accumulo tables. This is accomplished by prepending a
- * {@code tablePrefix} to every table that is owned by a Rya instance. Each
- * PCJ table is owned by a specific Rya instance, so it too must be prepended
- * with the instance's {@code tablePrefix}.
- * </p>
- * When Rya scans for PCJ tables that it may use when creating execution plans,
- * it looks for any table in Accumulo that has a name starting with its
- * {@code tablePrefix} immediately followed by "INDEX". Anything following
- * that portion of the table name is just a unique identifier for the SPARQL
- * query that is being precomputed. Here's an example of what a table name
- * may look like:
- * <pre>
- * demo_INDEX_QUERY:c8f5367c-1660-4210-a7cb-681ed004d2d9
- * </pre>
- * The "demo_INDEX" portion indicates this table is a PCJ table for the "demo_"
- * instance of Rya. The "_QUERY:c8f5367c-1660-4210-a7cb-681ed004d2d9" portion
- * could be anything at all that uniquely identifies the query that is being updated.
- *
- * @param tablePrefix - The Rya instance's table prefix. (not null)
- * @param uniqueId - The unique portion of the Rya PCJ table name. (not null)
- * @return A Rya PCJ table name build using the provided values.
- */
- public String makeTableName(final String tablePrefix, final String uniqueId) {
- return tablePrefix + "INDEX_" + uniqueId;
- }
- }
-
- /**
- * Create alternative variable orders for a SPARQL query based on
- * the original ordering of its results.
- */
- public static interface PcjVarOrderFactory {
-
- /**
- * Create alternative variable orders for a SPARQL query based on
- * the original ordering of its results.
- *
- * @param varOrder - The initial variable order of a SPARQL query. (not null)
- * @return A set of alternative variable orders for the original.
- */
- public Set<VariableOrder> makeVarOrders(VariableOrder varOrder);
- }
-
- /**
- * Shifts the variables to the left so that each variable will appear at
- * the head of the varOrder once.
- */
- @ParametersAreNonnullByDefault
- public static class ShiftVarOrderFactory implements PcjVarOrderFactory {
- @Override
- public Set<VariableOrder> makeVarOrders(final VariableOrder varOrder) {
- final Set<VariableOrder> varOrders = new HashSet<>();
-
- final List<String> cyclicBuff = Lists.newArrayList( varOrder.getVariableOrders() );
- final String[] varOrderBuff = new String[ cyclicBuff.size() ];
-
- for(int shift = 0; shift < cyclicBuff.size(); shift++) {
- // Build a variable order.
- for(int i = 0; i < cyclicBuff.size(); i++) {
- varOrderBuff[i] = cyclicBuff.get(i);
- }
- varOrders.add( new VariableOrder(varOrderBuff) );
-
- // Shift the order the variables will appear in the cyclic buffer.
- cyclicBuff.add( cyclicBuff.remove(0) );
- }
-
- return varOrders;
- }
- }
-
- /**
- * Indicates one of the {@link PcjTables} functions has failed to perform its task.
- */
- public static class PcjException extends Exception {
- private static final long serialVersionUID = 1L;
-
- /**
- * Constructs an instance of {@link PcjException}.
- *
- * @param message - Describes why the exception is being thrown.
- */
- public PcjException(final String message) {
- super(message);
- }
-
- /**
- * Constructs an instance of {@link PcjException}.
- *
- * @param message - Describes why the exception is being thrown.
- * @param cause - The exception that caused this one to be thrown.
- */
- public PcjException(final String message, final Throwable cause) {
- super(message, cause);
- }
- }
-
- /**
- * Create a new PCJ table within an Accumulo instance for a SPARQL query.
- * For example, calling the function like this:
- * <pre>
- * PcjTables.createPcjTable(
- * accumuloConn,
- *
- * "foo_INDEX_query1234",
- *
- * Sets.newHashSet(
- * new VariableOrder("city;worker;customer"),
- * new VariableOrder("worker;customer;city") ,
- * new VariableOrder("customer;city;worker")),
- *
- * "SELECT ?customer ?worker ?city { " +
- * "?customer <http://talksTo> ?worker. " +
- * "?worker <http://livesIn> ?city. " +
- * "?worker <http://worksAt> <http://Home>. " +
- * "}");
- * </pre>
- * </p>
- * Will result in an Accumulo table named "foo_INDEX_query1234" with the following entries:
- * <table border="1" style="width:100%">
- * <tr> <th>Row ID</td> <th>Column</td> <th>Value</td> </tr>
- * <tr> <td>pcjMetadata</td> <td>metadata:sparql</td> <td> ... UTF-8 bytes encoding the query string ... </td> </tr>
- * <tr> <td>pcjMetadata</td> <td>metadata:cardinality</td> <td> The query's cardinality </td> </tr>
- * <tr> <td>pcjMetadata</td> <td>metadata:variableOrders</td> <td> The variable orders the results are written to </td> </tr>
- * </table>
- *
- * @param accumuloConn - A connection to the Accumulo that hosts the PCJ table. (not null)
- * @param pcjTableName - The name of the table that will be created. (not null)
- * @param varOrders - The variable orders the results within the table will be written to. (not null)
- * @param sparql - The query this table's results solves. (not null)
- * @throws PcjException Could not create a new PCJ table either because Accumulo
- * would not let us create it or the PCJ metadata was not able to be written to it.
- */
- public void createPcjTable(
- final Connector accumuloConn,
- final String pcjTableName,
- final Set<VariableOrder> varOrders,
- final String sparql) throws PcjException {
- checkNotNull(accumuloConn);
- checkNotNull(pcjTableName);
- checkNotNull(varOrders);
- checkNotNull(sparql);
-
- final TableOperations tableOps = accumuloConn.tableOperations();
- if(!tableOps.exists(pcjTableName)) {
- try {
- // Create the new table in Accumulo.
- tableOps.create(pcjTableName);
-
- // Write the PCJ Metadata to the newly created table.
- final PcjMetadata pcjMetadata = new PcjMetadata(sparql, 0L, varOrders);
- final List<Mutation> mutations = makeWriteMetadataMutations(pcjMetadata);
-
- final BatchWriter writer = accumuloConn.createBatchWriter(pcjTableName, new BatchWriterConfig());
- writer.addMutations(mutations);
- writer.close();
- } catch (final TableExistsException e) {
- log.warn("Something else just created the Rya PCJ export table named '" + pcjTableName
- + "'. This is unexpected, but we will continue as normal.");
- } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
- throw new PcjException("Could not create a new PCJ named: " + pcjTableName, e);
- }
- }
- }
-
- /**
- * Create the {@link Mutation}s required to write a {@link PCJMetadata} object
- * to an Accumulo table.
- *
- * @param metadata - The metadata to write. (not null)
- * @return An ordered list of mutations that write the metadata to an Accumulo table.
- */
- private static List<Mutation> makeWriteMetadataMutations(final PcjMetadata metadata) {
- checkNotNull(metadata);
-
- final List<Mutation> mutations = new LinkedList<>();
-
- // SPARQL Query
- Mutation mutation = new Mutation(PCJ_METADATA_ROW_ID);
- final Value query = new Value( stringLexicoder.encode(metadata.getSparql()) );
- mutation.put(PCJ_METADATA_FAMILY, PCJ_METADATA_SPARQL_QUERY, query);
- mutations.add(mutation);
-
- // Cardinality
- mutation = new Mutation(PCJ_METADATA_ROW_ID);
- final Value cardinality = new Value( longLexicoder.encode(new Long(metadata.getCardinality())) );
- mutation.put(PCJ_METADATA_FAMILY, PCJ_METADATA_CARDINALITY, cardinality);
- mutations.add(mutation);
-
- // Variable Orders
- final List<String> varOrderStrings = new ArrayList<>();
- for(final VariableOrder varOrder : metadata.getVarOrders()) {
- varOrderStrings.add( varOrder.toString() );
- }
-
- mutation = new Mutation(PCJ_METADATA_ROW_ID);
- final Value variableOrders = new Value( listLexicoder.encode(varOrderStrings) );
- mutation.put(PCJ_METADATA_FAMILY, PCJ_METADATA_VARIABLE_ORDERS, variableOrders);
- mutations.add(mutation);
-
- return mutations;
- }
-
- /**
- * Fetch the {@link PCJMetadata} from an Accumulo table.
- * <p>
- * This method assumes the PCJ table has already been created.
- *
- * @param accumuloConn - A connection to the Accumulo that hosts the PCJ table. (not null)
- * @param pcjTableName - The name of the table that will be search. (not null)
- * @return The PCJ Metadata that has been stolred in the in the PCJ Table.
- * @throws PcjException The PCJ Table does not exist.
- */
- public PcjMetadata getPcjMetadata(
- final Connector accumuloConn,
- final String pcjTableName) throws PcjException {
- checkNotNull(accumuloConn);
- checkNotNull(pcjTableName);
-
- try {
- // Create an Accumulo scanner that iterates through the metadata entries.
- final Scanner scanner = accumuloConn.createScanner(pcjTableName, new Authorizations());
- final Iterator<Entry<Key, Value>> entries = scanner.iterator();
-
- // No metadata has been stored in the table yet.
- if(!entries.hasNext()) {
- throw new PcjException("Could not find any PCJ metadata in the table named: " + pcjTableName);
- }
-
- // Fetch the metadata from the entries. Assuming they all have the same cardinality and sparql query.
- String sparql = null;
- Long cardinality = null;
- final Set<VariableOrder> varOrders = new HashSet<>();
-
- while(entries.hasNext()) {
- final Entry<Key, Value> entry = entries.next();
- final Text columnQualifier = entry.getKey().getColumnQualifier();
- final byte[] value = entry.getValue().get();
-
- if(columnQualifier.equals(PCJ_METADATA_SPARQL_QUERY)) {
- sparql = stringLexicoder.decode(value);
- } else if(columnQualifier.equals(PCJ_METADATA_CARDINALITY)) {
- cardinality = longLexicoder.decode(value);
- } else if(columnQualifier.equals(PCJ_METADATA_VARIABLE_ORDERS)) {
- for(final String varOrderStr : listLexicoder.decode(value)) {
- varOrders.add( new VariableOrder(varOrderStr) );
- }
- }
- }
-
- return new PcjMetadata(sparql, cardinality, varOrders);
-
- } catch (final TableNotFoundException e) {
- throw new PcjException("Could not add results to a PCJ because the PCJ table does not exist.", e);
- }
- }
-
- /**
- * Add a collection of results to a PCJ table. The table's cardinality will
- * be updated to include the new results.
- * <p>
- * This method assumes the PCJ table has already been created.
- *
- * @param accumuloConn - A connection to the Accumulo that hosts the PCJ table. (not null)
- * @param pcjTableName - The name of the PCJ table that will receive the results. (not null)
- * @param results - Binding sets that will be written to the PCJ table. (not null)
- * @throws PcjException The provided PCJ table doesn't exist, is missing the
- * PCJ metadata, or the result could not be written to it.
- */
- public void addResults(
- final Connector accumuloConn,
- final String pcjTableName,
- final Collection<VisibilityBindingSet> results) throws PcjException {
- checkNotNull(accumuloConn);
- checkNotNull(pcjTableName);
- checkNotNull(results);
-
- // Write a result to each of the variable orders that are in the table.
- writeResults(accumuloConn, pcjTableName, results);
-
- // Increment the cardinality of the query by the number of new results.
- updateCardinality(accumuloConn, pcjTableName, results.size());
- }
-
- /**
- * Add a collection of results to a specific PCJ table.
- *
- * @param accumuloConn - A connection to the Accumulo that hosts the PCJ table. (not null)
- * @param pcjTableName - The name of the PCJ table that will receive the results. (not null)
- * @param results - Binding sets that will be written to the PCJ table. (not null)
- * @throws PcjException The provided PCJ table doesn't exist, is missing the
- * PCJ metadata, or the result could not be written to it.
- */
- private void writeResults(
- final Connector accumuloConn,
- final String pcjTableName,
- final Collection<VisibilityBindingSet> results) throws PcjException {
- checkNotNull(accumuloConn);
- checkNotNull(pcjTableName);
- checkNotNull(results);
-
- // Fetch the variable orders from the PCJ table.
- final PcjMetadata metadata = getPcjMetadata(accumuloConn, pcjTableName);
-
- // Write each result formatted using each of the variable orders.
- BatchWriter writer = null;
- try {
- writer = accumuloConn.createBatchWriter(pcjTableName, new BatchWriterConfig());
- for(final VisibilityBindingSet result : results) {
- final Set<Mutation> addResultMutations = makeWriteResultMutations(metadata.getVarOrders(), result);
- writer.addMutations( addResultMutations );
- }
- } catch (TableNotFoundException | MutationsRejectedException e) {
- throw new PcjException("Could not add results to the PCJ table named: " + pcjTableName, e);
- } finally {
- if(writer != null) {
- try {
- writer.close();
- } catch (final MutationsRejectedException e) {
- throw new PcjException("Could not add results to a PCJ table because some of the mutations were rejected.", e);
- }
- }
- }
- }
-
- /**
- * Create the {@link Mutations} required to write a new {@link BindingSet}
- * to a PCJ table for each {@link VariableOrder} that is provided.
- *
- * @param varOrders - The variables orders the result will be written to. (not null)
- * @param result - A new PCJ result. (not null)
- * @return Mutation that will write the result to a PCJ table.
- * @throws PcjException The binding set could not be encoded.
- */
- private static Set<Mutation> makeWriteResultMutations(
- final Set<VariableOrder> varOrders,
- final VisibilityBindingSet result) throws PcjException {
- checkNotNull(varOrders);
- checkNotNull(result);
-
- final Set<Mutation> mutations = new HashSet<>();
- final AccumuloPcjSerializer converter = new AccumuloPcjSerializer();
-
- for(final VariableOrder varOrder : varOrders) {
- try {
- // Serialize the result to the variable order.
- final byte[] serializedResult = converter.convert(result, varOrder);
-
- // Row ID = binding set values, Column Family = variable order of the binding set.
- final Mutation addResult = new Mutation(serializedResult);
- final String visibility = result.getVisibility();
- addResult.put(varOrder.toString(), "", new ColumnVisibility(visibility), "");
- mutations.add(addResult);
- } catch(final BindingSetConversionException e) {
- throw new PcjException("Could not serialize a result.", e);
- }
- }
-
- return mutations;
- }
-
- /**
- * Update the cardinality of a PCJ by a {@code delta}.
- *
- * @param accumuloConn - A connection to the Accumulo that hosts the PCJ table. (not null)
- * @param pcjTableName - The name of the PCJ table that will have its cardinality updated. (not null)
- * @param delta - How much the cardinality will change.
- * @throws PcjException The cardinality could not be updated.
- */
- private void updateCardinality(
- final Connector accumuloConn,
- final String pcjTableName,
- final long delta) throws PcjException {
- checkNotNull(accumuloConn);
- checkNotNull(pcjTableName);
-
- ConditionalWriter conditionalWriter = null;
- try {
- conditionalWriter = accumuloConn.createConditionalWriter(pcjTableName, new ConditionalWriterConfig());
-
- boolean updated = false;
- while(!updated) {
- // Write the conditional update request to Accumulo.
- final long cardinality = getPcjMetadata(accumuloConn, pcjTableName).getCardinality();
- final ConditionalMutation mutation = makeUpdateCardinalityMutation(cardinality, delta);
- final ConditionalWriter.Result result = conditionalWriter.write(mutation);
-
- // Interpret the result.
- switch(result.getStatus()) {
- case ACCEPTED:
- updated = true;
- break;
- case REJECTED:
- break;
- case UNKNOWN:
- // We do not know if the mutation succeeded. At best, we can hope the metadata hasn't been updated
- // since we originally fetched it and try again. Otherwise, continue forwards as if it worked. It's
- // okay if this number is slightly off.
- final long newCardinality = getPcjMetadata(accumuloConn, pcjTableName).getCardinality();
- if(newCardinality != cardinality) {
- updated = true;
- }
- break;
- case VIOLATED:
- throw new PcjException("The cardinality could not be updated because the commit violated a table constraint.");
- case INVISIBLE_VISIBILITY:
- throw new PcjException("The condition contains a visibility the updater can not satisfy.");
- }
- }
- } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
- throw new PcjException("Could not update the cardinality value of the PCJ Table named: " + pcjTableName, e);
- } finally {
- if(conditionalWriter != null) {
- conditionalWriter.close();
- }
- }
- }
-
- /**
- * Creates a {@link ConditionalMutation} that only updates the cardinality
- * of the PCJ table if the old value has not changed by the time this mutation
- * is committed to Accumulo.
- *
- * @param current - The current cardinality value.
- * @param delta - How much the cardinality will change.
- * @return The mutation that will perform the conditional update.
- */
- private static ConditionalMutation makeUpdateCardinalityMutation(final long current, final long delta) {
- // Try to update the cardinality by the delta.
- final ConditionalMutation mutation = new ConditionalMutation(PCJ_METADATA_ROW_ID);
- final Condition lastCardinalityStillCurrent = new Condition(
- PCJ_METADATA_FAMILY,
- PCJ_METADATA_CARDINALITY);
-
- // Require the old cardinality to be the value we just read.
- final byte[] currentCardinalityBytes = longLexicoder.encode( current );
- lastCardinalityStillCurrent.setValue( currentCardinalityBytes );
- mutation.addCondition(lastCardinalityStillCurrent);
-
- // If that is the case, then update to the new value.
- final Value newCardinality = new Value( longLexicoder.encode(current + delta) );
- mutation.put(PCJ_METADATA_FAMILY, PCJ_METADATA_CARDINALITY, newCardinality);
- return mutation;
- }
-
- /**
- * Scan Rya for results that solve the PCJ's query and store them in the PCJ table.
- * <p>
- * This method assumes the PCJ table has already been created.
- *
- * @param accumuloConn - A connection to the Accumulo that hosts the PCJ table. (not null)
- * @param pcjTableName - The name of the PCJ table that will receive the results. (not null)
- * @param ryaConn - A connection to the Rya store that will be queried to find results. (not null)
- * @throws PcjException If results could not be written to the PCJ table,
- * the PCJ table does not exist, or the query that is being execute
- * was malformed.
- */
- public void populatePcj(
- final Connector accumuloConn,
- final String pcjTableName,
- final RepositoryConnection ryaConn) throws PcjException {
- checkNotNull(accumuloConn);
- checkNotNull(pcjTableName);
- checkNotNull(ryaConn);
-
- try {
- // Fetch the query that needs to be executed from the PCJ table.
- final PcjMetadata pcjMetadata = getPcjMetadata(accumuloConn, pcjTableName);
- final String sparql = pcjMetadata.getSparql();
-
- // Query Rya for results to the SPARQL query.
- final TupleQuery query = ryaConn.prepareTupleQuery(QueryLanguage.SPARQL, sparql);
- final TupleQueryResult results = query.evaluate();
-
- // Load batches of 1000 of them at a time into the PCJ table
- final Set<VisibilityBindingSet> batch = new HashSet<>(1000);
- while(results.hasNext()) {
- batch.add( new VisibilityBindingSet(results.next()) );
-
- if(batch.size() == 1000) {
- addResults(accumuloConn, pcjTableName, batch);
- batch.clear();
- }
- }
-
- if(!batch.isEmpty()) {
- addResults(accumuloConn, pcjTableName, batch);
- }
-
- } catch (RepositoryException | MalformedQueryException | QueryEvaluationException e) {
- throw new PcjException("Could not populate a PCJ table with Rya results for the table named: " + pcjTableName, e);
- }
- }
-
- private static final PcjVarOrderFactory DEFAULT_VAR_ORDER_FACTORY = new ShiftVarOrderFactory();
-
- /**
- * Creates a new PCJ Table in Accumulo and populates it by scanning an
- * instance of Rya for historic matches.
- * <p>
- * If any portion of this operation fails along the way, the partially
- * create PCJ table will be left in Accumulo.
- *
- * @param ryaConn - Connects to the Rya that will be scanned. (not null)
- * @param accumuloConn - Connects to the accumulo that hosts the PCJ results. (not null)
- * @param pcjTableName - The name of the PCJ table that will be created. (not null)
- * @param sparql - The SPARQL query whose results will be loaded into the table. (not null)
- * @param resultVariables - The variables that are included in the query's resulting binding sets. (not null)
- * @param pcjVarOrderFactory - An optional factory that indicates the various variable orders
- * the results will be stored in. If one is not provided, then {@link ShiftVarOrderFactory}
- * is used by default. (not null)
- * @throws PcjException The PCJ table could not be create or the values from
- * Rya were not able to be loaded into it.
- */
- public void createAndPopulatePcj(
- final RepositoryConnection ryaConn,
- final Connector accumuloConn,
- final String pcjTableName,
- final String sparql,
- final String[] resultVariables,
- final Optional<PcjVarOrderFactory> pcjVarOrderFactory) throws PcjException {
- checkNotNull(ryaConn);
- checkNotNull(accumuloConn);
- checkNotNull(pcjTableName);
- checkNotNull(sparql);
- checkNotNull(resultVariables);
- checkNotNull(pcjVarOrderFactory);
-
- // Create the PCJ's variable orders.
- final PcjVarOrderFactory varOrderFactory = pcjVarOrderFactory.or(DEFAULT_VAR_ORDER_FACTORY);
- final Set<VariableOrder> varOrders = varOrderFactory.makeVarOrders( new VariableOrder(resultVariables) );
-
- // Create the PCJ table in Accumulo.
- createPcjTable(accumuloConn, pcjTableName, varOrders, sparql);
-
- // Load historic matches from Rya into the PCJ table.
- populatePcj(accumuloConn, pcjTableName, ryaConn);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/14073a23/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/VisibilityBindingSetStringConverter.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/VisibilityBindingSetStringConverter.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/VisibilityBindingSetStringConverter.java
deleted file mode 100644
index 832186c..0000000
--- a/extras/indexing/src/main/java/mvm/rya/indexing/external/tupleSet/VisibilityBindingSetStringConverter.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package mvm.rya.indexing.external.tupleSet;
-
-import javax.annotation.ParametersAreNonnullByDefault;
-
-import org.openrdf.query.BindingSet;
-
-import com.google.common.base.Strings;
-
-import mvm.rya.indexing.accumulo.VisibilityBindingSet;
-import mvm.rya.indexing.external.tupleSet.PcjTables.VariableOrder;
-
-/**
- * Converts {@link BindingSet}s to Strings and back again. The Strings do not
- * include the binding names and are ordered with a {@link VariableOrder}.
- */
-@ParametersAreNonnullByDefault
-public class VisibilityBindingSetStringConverter extends BindingSetStringConverter {
- public static final char VISIBILITY_DELIM = 1;
-
- @Override
- public String convert(final BindingSet bindingSet, final VariableOrder varOrder) {
- String visibility = "";
- if(bindingSet instanceof VisibilityBindingSet) {
- final VisibilityBindingSet visiSet = (VisibilityBindingSet) bindingSet;
- if(!Strings.isNullOrEmpty(visiSet.getVisibility())) {
- visibility = VISIBILITY_DELIM + visiSet.getVisibility();
- }
- }
- return super.convert(bindingSet, varOrder) + visibility;
- }
-
- @Override
- public BindingSet convert(final String bindingSetString, final VariableOrder varOrder) {
- final String[] visiStrings = bindingSetString.split("" + VISIBILITY_DELIM);
- BindingSet bindingSet = super.convert(visiStrings[0], varOrder);
-
- if(visiStrings.length > 1) {
- bindingSet = new VisibilityBindingSet(bindingSet, visiStrings[1]);
- } else {
- bindingSet = new VisibilityBindingSet(bindingSet);
- }
- return bindingSet;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/14073a23/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloConstantPcjIntegrationTest.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloConstantPcjIntegrationTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloConstantPcjIntegrationTest.java
index 491c39d..8041f16 100644
--- a/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloConstantPcjIntegrationTest.java
+++ b/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloConstantPcjIntegrationTest.java
@@ -21,11 +21,6 @@ package mvm.rya.indexing.external;
import java.util.List;
-import mvm.rya.api.persist.RyaDAOException;
-import mvm.rya.indexing.external.tupleSet.PcjTables.PcjException;
-import mvm.rya.indexing.external.tupleSet.PcjTables.PcjVarOrderFactory;
-import mvm.rya.rdftriplestore.inference.InferenceEngineException;
-
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
@@ -33,6 +28,8 @@ import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.rya.indexing.pcj.storage.PcjException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -56,6 +53,9 @@ import org.openrdf.sail.SailException;
import com.google.common.base.Optional;
+import mvm.rya.api.persist.RyaDAOException;
+import mvm.rya.rdftriplestore.inference.InferenceEngineException;
+
public class AccumuloConstantPcjIntegrationTest {
private SailRepositoryConnection conn, pcjConn;
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/14073a23/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloPcjIntegrationTest.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloPcjIntegrationTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloPcjIntegrationTest.java
index a13bea9..d2b19f0 100644
--- a/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloPcjIntegrationTest.java
+++ b/extras/indexing/src/test/java/mvm/rya/indexing/external/AccumuloPcjIntegrationTest.java
@@ -24,14 +24,6 @@ import java.util.Collection;
import java.util.List;
import java.util.Set;
-import mvm.rya.api.persist.RyaDAOException;
-import mvm.rya.indexing.IndexPlanValidator.IndexPlanValidator;
-import mvm.rya.indexing.external.tupleSet.AccumuloIndexSet;
-import mvm.rya.indexing.external.tupleSet.ExternalTupleSet;
-import mvm.rya.indexing.external.tupleSet.PcjTables.PcjException;
-import mvm.rya.indexing.external.tupleSet.PcjTables.PcjVarOrderFactory;
-import mvm.rya.rdftriplestore.inference.InferenceEngineException;
-
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
@@ -41,6 +33,8 @@ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.rya.indexing.pcj.storage.PcjException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -69,6 +63,12 @@ import com.beust.jcommander.internal.Sets;
import com.google.common.base.Optional;
import com.google.common.collect.Lists;
+import mvm.rya.api.persist.RyaDAOException;
+import mvm.rya.indexing.IndexPlanValidator.IndexPlanValidator;
+import mvm.rya.indexing.external.tupleSet.AccumuloIndexSet;
+import mvm.rya.indexing.external.tupleSet.ExternalTupleSet;
+import mvm.rya.rdftriplestore.inference.InferenceEngineException;
+
public class AccumuloPcjIntegrationTest {
private SailRepositoryConnection conn, pcjConn;
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/14073a23/extras/indexing/src/test/java/mvm/rya/indexing/external/PCJOptionalTestIT.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/external/PCJOptionalTestIT.java b/extras/indexing/src/test/java/mvm/rya/indexing/external/PCJOptionalTestIT.java
index e3ef741..e3c6fa6 100644
--- a/extras/indexing/src/test/java/mvm/rya/indexing/external/PCJOptionalTestIT.java
+++ b/extras/indexing/src/test/java/mvm/rya/indexing/external/PCJOptionalTestIT.java
@@ -21,15 +21,6 @@ package mvm.rya.indexing.external;
import java.util.ArrayList;
import java.util.List;
-import mvm.rya.api.persist.RyaDAOException;
-import mvm.rya.indexing.external.PrecompJoinOptimizerIntegrationTest.CountingResultHandler;
-import mvm.rya.indexing.external.PrecompJoinOptimizerTest.NodeCollector;
-import mvm.rya.indexing.external.tupleSet.ExternalTupleSet;
-import mvm.rya.indexing.external.tupleSet.PcjTables.PcjException;
-import mvm.rya.indexing.external.tupleSet.PcjTables.PcjVarOrderFactory;
-import mvm.rya.indexing.external.tupleSet.SimpleExternalTupleSet;
-import mvm.rya.rdftriplestore.inference.InferenceEngineException;
-
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
@@ -37,6 +28,8 @@ import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.rya.indexing.pcj.storage.PcjException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -64,6 +57,13 @@ import org.openrdf.sail.SailException;
import com.beust.jcommander.internal.Lists;
import com.google.common.base.Optional;
+import mvm.rya.api.persist.RyaDAOException;
+import mvm.rya.indexing.external.PrecompJoinOptimizerIntegrationTest.CountingResultHandler;
+import mvm.rya.indexing.external.PrecompJoinOptimizerTest.NodeCollector;
+import mvm.rya.indexing.external.tupleSet.ExternalTupleSet;
+import mvm.rya.indexing.external.tupleSet.SimpleExternalTupleSet;
+import mvm.rya.rdftriplestore.inference.InferenceEngineException;
+
public class PCJOptionalTestIT {
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/14073a23/extras/indexing/src/test/java/mvm/rya/indexing/external/PcjIntegrationTestingUtil.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/external/PcjIntegrationTestingUtil.java b/extras/indexing/src/test/java/mvm/rya/indexing/external/PcjIntegrationTestingUtil.java
index 1be88e2..3b87179 100644
--- a/extras/indexing/src/test/java/mvm/rya/indexing/external/PcjIntegrationTestingUtil.java
+++ b/extras/indexing/src/test/java/mvm/rya/indexing/external/PcjIntegrationTestingUtil.java
@@ -8,9 +8,9 @@ package mvm.rya.indexing.external;
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,22 +25,6 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
-import mvm.rya.accumulo.AccumuloRdfConfiguration;
-import mvm.rya.api.persist.RyaDAOException;
-import mvm.rya.api.resolver.RyaTypeResolverException;
-import mvm.rya.sail.config.RyaSailFactory;
-import mvm.rya.indexing.accumulo.ConfigUtils;
-import mvm.rya.indexing.external.tupleSet.AccumuloPcjSerializer;
-import mvm.rya.indexing.external.tupleSet.BindingSetConverter.BindingSetConversionException;
-import mvm.rya.indexing.external.tupleSet.ExternalTupleSet;
-import mvm.rya.indexing.external.tupleSet.PcjTables;
-import mvm.rya.indexing.external.tupleSet.PcjTables.PcjException;
-import mvm.rya.indexing.external.tupleSet.PcjTables.PcjMetadata;
-import mvm.rya.indexing.external.tupleSet.PcjTables.PcjVarOrderFactory;
-import mvm.rya.indexing.external.tupleSet.PcjTables.ShiftVarOrderFactory;
-import mvm.rya.indexing.external.tupleSet.PcjTables.VariableOrder;
-import mvm.rya.rdftriplestore.inference.InferenceEngineException;
-
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchWriter;
@@ -50,6 +34,14 @@ import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.rya.indexing.pcj.storage.PcjException;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjSerializer;
+import org.apache.rya.indexing.pcj.storage.accumulo.BindingSetConverter.BindingSetConversionException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjTables;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
import org.openrdf.query.BindingSet;
import org.openrdf.query.MalformedQueryException;
import org.openrdf.query.QueryEvaluationException;
@@ -69,10 +61,17 @@ import org.openrdf.sail.Sail;
import com.google.common.base.Optional;
import com.google.common.collect.Sets;
+import mvm.rya.accumulo.AccumuloRdfConfiguration;
+import mvm.rya.api.persist.RyaDAOException;
+import mvm.rya.indexing.accumulo.ConfigUtils;
+import mvm.rya.indexing.external.tupleSet.ExternalTupleSet;
+import mvm.rya.rdftriplestore.inference.InferenceEngineException;
+import mvm.rya.sail.config.RyaSailFactory;
+
public class PcjIntegrationTestingUtil {
private static final AccumuloPcjSerializer converter = new AccumuloPcjSerializer();
-
+
public static Set<QueryModelNode> getTupleSets(TupleExpr te) {
final ExternalTupleVisitor etv = new ExternalTupleVisitor();
te.visit(etv);
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/14073a23/extras/indexing/src/test/java/mvm/rya/indexing/external/PrecompJoinOptimizerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/external/PrecompJoinOptimizerIntegrationTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/external/PrecompJoinOptimizerIntegrationTest.java
index 324dac5..f3e166d 100644
--- a/extras/indexing/src/test/java/mvm/rya/indexing/external/PrecompJoinOptimizerIntegrationTest.java
+++ b/extras/indexing/src/test/java/mvm/rya/indexing/external/PrecompJoinOptimizerIntegrationTest.java
@@ -21,11 +21,6 @@ package mvm.rya.indexing.external;
import java.util.List;
-import mvm.rya.api.persist.RyaDAOException;
-import mvm.rya.indexing.external.tupleSet.PcjTables.PcjException;
-import mvm.rya.indexing.external.tupleSet.PcjTables.PcjVarOrderFactory;
-import mvm.rya.rdftriplestore.inference.InferenceEngineException;
-
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
@@ -33,6 +28,8 @@ import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.rya.indexing.pcj.storage.PcjException;
+import org.apache.rya.indexing.pcj.storage.accumulo.PcjVarOrderFactory;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -56,6 +53,9 @@ import org.openrdf.sail.SailException;
import com.google.common.base.Optional;
+import mvm.rya.api.persist.RyaDAOException;
+import mvm.rya.rdftriplestore.inference.InferenceEngineException;
+
public class PrecompJoinOptimizerIntegrationTest {
private SailRepositoryConnection conn, pcjConn;