You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rya.apache.org by ca...@apache.org on 2017/08/18 18:51:26 UTC
[1/5] incubator-rya git commit: RYA-282-Nested-Query. Closes #192.
Repository: incubator-rya
Updated Branches:
refs/heads/master 6ce0b00b4 -> e387818ba
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/QueryIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/QueryIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/QueryIT.java
index 85c5030..6ecec02 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/QueryIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/QueryIT.java
@@ -36,7 +36,7 @@ import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.core.client.FluoClientImpl;
import org.apache.rya.api.client.RyaClient;
import org.apache.rya.api.client.accumulo.AccumuloRyaClientFactory;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePcj;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
@@ -68,25 +68,25 @@ import com.google.common.collect.Sets;
*/
public class QueryIT extends RyaExportITBase {
- private enum ExporterType {Pcj, Periodic};
-
+ private enum ExporterType {
+ Pcj, Periodic
+ };
+
@Test
public void optionalStatements() throws Exception {
// A query that has optional statement patterns. This query is looking for all
// people who have Law degrees and any BAR exams they have passed (though they
// do not have to have passed any).
- final String sparql =
- "SELECT ?person ?exam " +
- "WHERE {" +
- "?person <http://hasDegreeIn> <http://Law> . " +
- "OPTIONAL {?person <http://passedExam> ?exam } . " +
- "}";
+ final String sparql = "SELECT ?person ?exam " + "WHERE {" + "?person <http://hasDegreeIn> <http://Law> . "
+ + "OPTIONAL {?person <http://passedExam> ?exam } . " + "}";
// Create the Statements that will be loaded into Rya.
final ValueFactory vf = new ValueFactoryImpl();
final Collection<Statement> statements = Sets.newHashSet(
- vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://hasDegreeIn"), vf.createURI("http://Computer Science")),
- vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://passedExam"), vf.createURI("http://Certified Ethical Hacker")),
+ vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://hasDegreeIn"),
+ vf.createURI("http://Computer Science")),
+ vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://passedExam"),
+ vf.createURI("http://Certified Ethical Hacker")),
vf.createStatement(vf.createURI("http://Bob"), vf.createURI("http://hasDegreeIn"), vf.createURI("http://Law")),
vf.createStatement(vf.createURI("http://Bob"), vf.createURI("http://passedExam"), vf.createURI("http://MBE")),
vf.createStatement(vf.createURI("http://Bob"), vf.createURI("http://passedExam"), vf.createURI("http://BAR-Kansas")),
@@ -121,16 +121,10 @@ public class QueryIT extends RyaExportITBase {
// A query that find people who live in the USA, have been recruited by Geek Squad,
// and are skilled with computers. The resulting binding set includes everybody who
// was involved in the recruitment process.
- final String sparql =
- "SELECT ?recruiter ?candidate ?leader " +
- "{ " +
- "?recruiter <http://recruiterFor> <http://GeekSquad>. " +
- "?candidate <http://skilledWith> <http://Computers>. " +
- "?candidate <http://livesIn> \"USA\". " +
- "?leader <http://leaderOf> <http://GeekSquad>. " +
- "?recruiter <http://talksTo> ?candidate. " +
- "?candidate <http://talksTo> ?leader. " +
- "}";
+ final String sparql = "SELECT ?recruiter ?candidate ?leader " + "{ " + "?recruiter <http://recruiterFor> <http://GeekSquad>. "
+ + "?candidate <http://skilledWith> <http://Computers>. " + "?candidate <http://livesIn> \"USA\". "
+ + "?leader <http://leaderOf> <http://GeekSquad>. " + "?recruiter <http://talksTo> ?candidate. "
+ + "?candidate <http://talksTo> ?leader. " + "}";
// Create the Statements that will be loaded into Rya.
final ValueFactory vf = new ValueFactoryImpl();
@@ -139,11 +133,11 @@ public class QueryIT extends RyaExportITBase {
vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://leaderOf"), vf.createURI("http://GeekSquad")),
vf.createStatement(vf.createURI("http://Bob"), vf.createURI("http://leaderOf"), vf.createURI("http://GeekSquad")),
- // Recruiters
+ // Recruiters
vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://recruiterFor"), vf.createURI("http://GeekSquad")),
vf.createStatement(vf.createURI("http://David"), vf.createURI("http://recruiterFor"), vf.createURI("http://GeekSquad")),
- // Candidates
+ // Candidates
vf.createStatement(vf.createURI("http://Eve"), vf.createURI("http://skilledWith"), vf.createURI("http://Computers")),
vf.createStatement(vf.createURI("http://Eve"), vf.createURI("http://livesIn"), vf.createLiteral("USA")),
vf.createStatement(vf.createURI("http://Frank"), vf.createURI("http://skilledWith"), vf.createURI("http://Computers")),
@@ -155,7 +149,7 @@ public class QueryIT extends RyaExportITBase {
vf.createStatement(vf.createURI("http://Ivan"), vf.createURI("http://skilledWith"), vf.createURI("http://Computers")),
vf.createStatement(vf.createURI("http://Ivan"), vf.createURI("http://livesIn"), vf.createLiteral("USA")),
- // Candidates the recruiters talk to.
+ // Candidates the recruiters talk to.
vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://talksTo"), vf.createURI("http://Eve")),
vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://talksTo"), vf.createURI("http://George")),
vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://talksTo"), vf.createURI("http://Harry")),
@@ -163,7 +157,7 @@ public class QueryIT extends RyaExportITBase {
vf.createStatement(vf.createURI("http://David"), vf.createURI("http://talksTo"), vf.createURI("http://Frank")),
vf.createStatement(vf.createURI("http://David"), vf.createURI("http://talksTo"), vf.createURI("http://Ivan")),
- // Recruits that talk to leaders.
+ // Recruits that talk to leaders.
vf.createStatement(vf.createURI("http://Eve"), vf.createURI("http://talksTo"), vf.createURI("http://Alice")),
vf.createStatement(vf.createURI("http://George"), vf.createURI("http://talksTo"), vf.createURI("http://Alice")),
vf.createStatement(vf.createURI("http://Harry"), vf.createURI("http://talksTo"), vf.createURI("http://Bob")),
@@ -196,15 +190,9 @@ public class QueryIT extends RyaExportITBase {
@Test
public void withURIFilters() throws Exception {
- final String sparql =
- "SELECT ?customer ?worker ?city " +
- "{ " +
- "FILTER(?customer = <http://Alice>) " +
- "FILTER(?city = <http://London>) " +
- "?customer <http://talksTo> ?worker. " +
- "?worker <http://livesIn> ?city. " +
- "?worker <http://worksAt> <http://Chipotle>. " +
- "}";
+ final String sparql = "SELECT ?customer ?worker ?city " + "{ " + "FILTER(?customer = <http://Alice>) "
+ + "FILTER(?city = <http://London>) " + "?customer <http://talksTo> ?worker. " + "?worker <http://livesIn> ?city. "
+ + "?worker <http://worksAt> <http://Chipotle>. " + "}";
// Create the Statements that will be loaded into Rya.
final ValueFactory vf = new ValueFactoryImpl();
@@ -213,19 +201,19 @@ public class QueryIT extends RyaExportITBase {
vf.createStatement(vf.createURI("http://Bob"), vf.createURI("http://livesIn"), vf.createURI("http://London")),
vf.createStatement(vf.createURI("http://Bob"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")),
- vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://talksTo"), vf.createURI("http://Charlie")),
+ vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://talksTo"), vf.createURI("http://Charlie")),
vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://livesIn"), vf.createURI("http://London")),
vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")),
- vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://talksTo"), vf.createURI("http://David")),
+ vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://talksTo"), vf.createURI("http://David")),
vf.createStatement(vf.createURI("http://David"), vf.createURI("http://livesIn"), vf.createURI("http://London")),
vf.createStatement(vf.createURI("http://David"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")),
- vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://talksTo"), vf.createURI("http://Eve")),
+ vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://talksTo"), vf.createURI("http://Eve")),
vf.createStatement(vf.createURI("http://Eve"), vf.createURI("http://livesIn"), vf.createURI("http://Leeds")),
vf.createStatement(vf.createURI("http://Eve"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")),
- vf.createStatement(vf.createURI("http://Frank"), vf.createURI("http://talksTo"), vf.createURI("http://Alice")),
+ vf.createStatement(vf.createURI("http://Frank"), vf.createURI("http://talksTo"), vf.createURI("http://Alice")),
vf.createStatement(vf.createURI("http://Frank"), vf.createURI("http://livesIn"), vf.createURI("http://London")),
vf.createStatement(vf.createURI("http://Frank"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")));
@@ -256,13 +244,8 @@ public class QueryIT extends RyaExportITBase {
@Test
public void withNumericFilters() throws Exception {
- final String sparql =
- "SELECT ?name ?age " +
- "{" +
- "FILTER(?age < 30) ." +
- "?name <http://hasAge> ?age." +
- "?name <http://playsSport> \"Soccer\" " +
- "}";
+ final String sparql = "SELECT ?name ?age " + "{" + "FILTER(?age < 30) ." + "?name <http://hasAge> ?age."
+ + "?name <http://playsSport> \"Soccer\" " + "}";
// Create the Statements that will be loaded into Rya.
final ValueFactory vf = new ValueFactoryImpl();
@@ -273,7 +256,7 @@ public class QueryIT extends RyaExportITBase {
vf.createStatement(vf.createURI("http://David"), vf.createURI("http://hasAge"), vf.createLiteral(16)),
vf.createStatement(vf.createURI("http://Eve"), vf.createURI("http://hasAge"), vf.createLiteral(35)),
- vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://playsSport"), vf.createLiteral("Soccer")),
+ vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://playsSport"), vf.createLiteral("Soccer")),
vf.createStatement(vf.createURI("http://Bob"), vf.createURI("http://playsSport"), vf.createLiteral("Soccer")),
vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://playsSport"), vf.createLiteral("Basketball")),
vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://playsSport"), vf.createLiteral("Soccer")),
@@ -298,14 +281,8 @@ public class QueryIT extends RyaExportITBase {
@Test
public void withCustomFilters() throws Exception {
- final String sparql =
- "prefix ryafunc: <tag:rya.apache.org,2017:function#> " +
- "SELECT ?name ?age " +
- "{ " +
- "FILTER( ryafunc:isTeen(?age) ) . " +
- "?name <http://hasAge> ?age . " +
- "?name <http://playsSport> \"Soccer\" . " +
- "}";
+ final String sparql = "prefix ryafunc: <tag:rya.apache.org,2017:function#> " + "SELECT ?name ?age " + "{ "
+ + "FILTER( ryafunc:isTeen(?age) ) . " + "?name <http://hasAge> ?age . " + "?name <http://playsSport> \"Soccer\" . " + "}";
// Register a custom Filter.
final Function fooFunction = new Function() {
@@ -335,10 +312,12 @@ public class QueryIT extends RyaExportITBase {
final double doubleValue = literal.doubleValue();
return BooleanLiteralImpl.valueOf(doubleValue < TEEN_THRESHOLD);
} else {
- throw new ValueExprEvaluationException("unexpected datatype (expect decimal/int or floating) for function operand: " + args[0]);
+ throw new ValueExprEvaluationException(
+ "unexpected datatype (expect decimal/int or floating) for function operand: " + args[0]);
}
} else {
- throw new ValueExprEvaluationException("unexpected input value (expect non-null and numeric) for function: " + args[0]);
+ throw new ValueExprEvaluationException(
+ "unexpected input value (expect non-null and numeric) for function: " + args[0]);
}
} else {
throw new ValueExprEvaluationException("unexpected input value (expect literal) for function: " + args[0]);
@@ -358,7 +337,7 @@ public class QueryIT extends RyaExportITBase {
vf.createStatement(vf.createURI("http://David"), vf.createURI("http://hasAge"), vf.createLiteral(16)),
vf.createStatement(vf.createURI("http://Eve"), vf.createURI("http://hasAge"), vf.createLiteral(35)),
- vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://playsSport"), vf.createLiteral("Soccer")),
+ vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://playsSport"), vf.createLiteral("Soccer")),
vf.createStatement(vf.createURI("http://Bob"), vf.createURI("http://playsSport"), vf.createLiteral("Soccer")),
vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://playsSport"), vf.createLiteral("Basketball")),
vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://playsSport"), vf.createLiteral("Soccer")),
@@ -387,29 +366,32 @@ public class QueryIT extends RyaExportITBase {
final String dtPredUri = "http://www.w3.org/2006/time#inXSDDateTime";
final String dtPred = "<" + dtPredUri + ">";
- final String sparql =
- "PREFIX time: <http://www.w3.org/2006/time#> " +
- "PREFIX xml: <http://www.w3.org/2001/XMLSchema#> " +
- "PREFIX tempo: <tag:rya-rdf.org,2015:temporal#> " +
- "SELECT ?event ?time " +
- "WHERE { " +
- "?event " + dtPred + " ?time . " +
- "FILTER(?time > '2001-01-01T01:01:03-08:00'^^xml:dateTime) " +
- "}";
+ final String sparql = "PREFIX time: <http://www.w3.org/2006/time#> " + "PREFIX xml: <http://www.w3.org/2001/XMLSchema#> "
+ + "PREFIX tempo: <tag:rya-rdf.org,2015:temporal#> " + "SELECT ?event ?time " + "WHERE { " + "?event " + dtPred + " ?time . "
+ + "FILTER(?time > '2001-01-01T01:01:03-08:00'^^xml:dateTime) " + "}";
// Create the Statements that will be loaded into Rya.
final ValueFactory vf = new ValueFactoryImpl();
final DatatypeFactory dtf = DatatypeFactory.newInstance();
final Collection<Statement> statements = Sets.newHashSet(
- vf.createStatement(vf.createURI("http://eventz"), vf.createURI("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), vf.createURI("http://www.w3.org/2006/time#Instant")),
- vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri), vf.createLiteral(dtf.newXMLGregorianCalendar("2001-01-01T01:01:01-08:00"))), // 1 second
- vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri), vf.createLiteral(dtf.newXMLGregorianCalendar("2001-01-01T04:01:02.000-05:00"))), // 2 second
- vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri), vf.createLiteral(dtf.newXMLGregorianCalendar("2001-01-01T01:01:03-08:00"))), // 3 seconds
- vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri), vf.createLiteral(dtf.newXMLGregorianCalendar("2001-01-01T01:01:04-08:00"))), // 4 seconds
- vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri), vf.createLiteral(dtf.newXMLGregorianCalendar("2001-01-01T09:01:05Z"))), // 5 seconds
- vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri), vf.createLiteral(dtf.newXMLGregorianCalendar("2006-01-01T05:00:00.000Z"))),
- vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri), vf.createLiteral(dtf.newXMLGregorianCalendar("2007-01-01T05:00:00.000Z"))),
- vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri), vf.createLiteral(dtf.newXMLGregorianCalendar("2008-01-01T05:00:00.000Z"))));
+ vf.createStatement(vf.createURI("http://eventz"), vf.createURI("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"),
+ vf.createURI("http://www.w3.org/2006/time#Instant")),
+ vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri),
+ vf.createLiteral(dtf.newXMLGregorianCalendar("2001-01-01T01:01:01-08:00"))), // 1 second
+ vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri),
+ vf.createLiteral(dtf.newXMLGregorianCalendar("2001-01-01T04:01:02.000-05:00"))), // 2 second
+ vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri),
+ vf.createLiteral(dtf.newXMLGregorianCalendar("2001-01-01T01:01:03-08:00"))), // 3 seconds
+ vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri),
+ vf.createLiteral(dtf.newXMLGregorianCalendar("2001-01-01T01:01:04-08:00"))), // 4 seconds
+ vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri),
+ vf.createLiteral(dtf.newXMLGregorianCalendar("2001-01-01T09:01:05Z"))), // 5 seconds
+ vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri),
+ vf.createLiteral(dtf.newXMLGregorianCalendar("2006-01-01T05:00:00.000Z"))),
+ vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri),
+ vf.createLiteral(dtf.newXMLGregorianCalendar("2007-01-01T05:00:00.000Z"))),
+ vf.createStatement(vf.createURI("http://eventz"), vf.createURI(dtPredUri),
+ vf.createLiteral(dtf.newXMLGregorianCalendar("2008-01-01T05:00:00.000Z"))));
// Create the expected results of the SPARQL query once the PCJ has been computed.
final Set<BindingSet> expectedResults = new HashSet<>();
@@ -442,97 +424,99 @@ public class QueryIT extends RyaExportITBase {
// Verify the end results of the query match the expected results.
runTest(sparql, statements, expectedResults, ExporterType.Pcj);
}
-
-
+
@Test
public void periodicQueryTestWithoutAggregation() throws Exception {
- String query = "prefix function: <http://org.apache.rya/function#> " //n
- + "prefix time: <http://www.w3.org/2006/time#> " //n
- + "select ?id where {" //n
- + "Filter(function:periodic(?time, 2, .5, time:hours)) " //n
- + "?obs <uri:hasTime> ?time. " //n
- + "?obs <uri:hasId> ?id }"; //n
+ String query = "prefix function: <http://org.apache.rya/function#> " // n
+ + "prefix time: <http://www.w3.org/2006/time#> " // n
+ + "select ?id where {" // n
+ + "Filter(function:periodic(?time, 2, .5, time:hours)) " // n
+ + "?obs <uri:hasTime> ?time. " // n
+ + "?obs <uri:hasId> ?id }"; // n
// Create the Statements that will be loaded into Rya.
final ValueFactory vf = new ValueFactoryImpl();
final DatatypeFactory dtf = DatatypeFactory.newInstance();
ZonedDateTime time = ZonedDateTime.now();
long currentTime = time.toInstant().toEpochMilli();
-
+
ZonedDateTime zTime1 = time.minusMinutes(30);
String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
-
+
ZonedDateTime zTime2 = zTime1.minusMinutes(30);
String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
-
+
ZonedDateTime zTime3 = zTime2.minusMinutes(30);
String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
-
+
ZonedDateTime zTime4 = zTime3.minusMinutes(30);
String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT);
-
+
final Collection<Statement> statements = Sets.newHashSet(
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")),
- vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")),
- vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
- vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasId"), vf.createLiteral("id_4"))
- );
+ vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
+ vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasId"), vf.createLiteral("id_4")));
// Create the expected results of the SPARQL query once the PCJ has been computed.
final Set<BindingSet> expectedResults = new HashSet<>();
long period = 1800000;
- long binId = (currentTime/period)*period;
-
+ long binId = (currentTime / period) * period;
+
MapBindingSet bs = new MapBindingSet();
bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId + period));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(binId + 2*period));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId + 2 * period));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(binId + 3*period));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId + 3 * period));
expectedResults.add(bs);
bs = new MapBindingSet();
bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId + period));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(binId + 2*period));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId + 2 * period));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("id", vf.createLiteral("id_3", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("id", vf.createLiteral("id_3", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId + period));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("id", vf.createLiteral("id_4", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId));
@@ -541,183 +525,189 @@ public class QueryIT extends RyaExportITBase {
// Verify the end results of the query match the expected results.
runTest(query, statements, expectedResults, ExporterType.Periodic);
}
-
-
+
@Test
public void periodicQueryTestWithAggregation() throws Exception {
- String query = "prefix function: <http://org.apache.rya/function#> " //n
- + "prefix time: <http://www.w3.org/2006/time#> " //n
- + "select (count(?obs) as ?total) where {" //n
- + "Filter(function:periodic(?time, 2, .5, time:hours)) " //n
- + "?obs <uri:hasTime> ?time. " //n
- + "?obs <uri:hasId> ?id }"; //n
+ String query = "prefix function: <http://org.apache.rya/function#> " // n
+ + "prefix time: <http://www.w3.org/2006/time#> " // n
+ + "select (count(?obs) as ?total) where {" // n
+ + "Filter(function:periodic(?time, 2, .5, time:hours)) " // n
+ + "?obs <uri:hasTime> ?time. " // n
+ + "?obs <uri:hasId> ?id }"; // n
// Create the Statements that will be loaded into Rya.
final ValueFactory vf = new ValueFactoryImpl();
final DatatypeFactory dtf = DatatypeFactory.newInstance();
ZonedDateTime time = ZonedDateTime.now();
long currentTime = time.toInstant().toEpochMilli();
-
+
ZonedDateTime zTime1 = time.minusMinutes(30);
String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
-
+
ZonedDateTime zTime2 = zTime1.minusMinutes(30);
String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
-
+
ZonedDateTime zTime3 = zTime2.minusMinutes(30);
String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
-
+
ZonedDateTime zTime4 = zTime3.minusMinutes(30);
String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT);
-
+
final Collection<Statement> statements = Sets.newHashSet(
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")),
- vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")),
- vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
- vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasId"), vf.createLiteral("id_4"))
- );
+ vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
+ vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasId"), vf.createLiteral("id_4")));
// Create the expected results of the SPARQL query once the PCJ has been computed.
final Set<BindingSet> expectedResults = new HashSet<>();
long period = 1800000;
- long binId = (currentTime/period)*period;
-
+ long binId = (currentTime / period) * period;
+
MapBindingSet bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("4", XMLSchema.INTEGER));
bs.addBinding("periodicBinId", vf.createLiteral(binId));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("3", XMLSchema.INTEGER));
bs.addBinding("periodicBinId", vf.createLiteral(binId + period));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
- bs.addBinding("periodicBinId", vf.createLiteral(binId + 2*period));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId + 2 * period));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
- bs.addBinding("periodicBinId", vf.createLiteral(binId + 3*period));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId + 3 * period));
expectedResults.add(bs);
-
// Verify the end results of the query match the expected results.
runTest(query, statements, expectedResults, ExporterType.Periodic);
}
-
+
@Test
public void periodicQueryTestWithAggregationAndGroupBy() throws Exception {
- String query = "prefix function: <http://org.apache.rya/function#> " //n
- + "prefix time: <http://www.w3.org/2006/time#> " //n
- + "select ?id (count(?obs) as ?total) where {" //n
- + "Filter(function:periodic(?time, 2, .5, time:hours)) " //n
- + "?obs <uri:hasTime> ?time. " //n
- + "?obs <uri:hasId> ?id } group by ?id"; //n
+ String query = "prefix function: <http://org.apache.rya/function#> " // n
+ + "prefix time: <http://www.w3.org/2006/time#> " // n
+ + "select ?id (count(?obs) as ?total) where {" // n
+ + "Filter(function:periodic(?time, 2, .5, time:hours)) " // n
+ + "?obs <uri:hasTime> ?time. " // n
+ + "?obs <uri:hasId> ?id } group by ?id"; // n
// Create the Statements that will be loaded into Rya.
final ValueFactory vf = new ValueFactoryImpl();
final DatatypeFactory dtf = DatatypeFactory.newInstance();
ZonedDateTime time = ZonedDateTime.now();
long currentTime = time.toInstant().toEpochMilli();
-
+
ZonedDateTime zTime1 = time.minusMinutes(30);
String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
-
+
ZonedDateTime zTime2 = zTime1.minusMinutes(30);
String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
-
+
ZonedDateTime zTime3 = zTime2.minusMinutes(30);
String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
-
+
ZonedDateTime zTime4 = zTime3.minusMinutes(30);
String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT);
-
+
final Collection<Statement> statements = Sets.newHashSet(
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")),
- vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")),
- vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
+ vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasId"), vf.createLiteral("id_4")),
- vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
- vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2"))
- );
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")));
// Create the expected results of the SPARQL query once the PCJ has been computed.
final Set<BindingSet> expectedResults = new HashSet<>();
long period = 1800000;
- long binId = (currentTime/period)*period;
-
+ long binId = (currentTime / period) * period;
+
MapBindingSet bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
bs.addBinding("id", vf.createLiteral("id_3", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
bs.addBinding("id", vf.createLiteral("id_4", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId + period));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId + period));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
bs.addBinding("id", vf.createLiteral("id_3", XMLSchema.STRING));
bs.addBinding("periodicBinId", vf.createLiteral(binId + period));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(binId + 2*period));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId + 2 * period));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(binId + 2*period));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId + 2 * period));
expectedResults.add(bs);
-
+
bs = new MapBindingSet();
bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
- bs.addBinding("periodicBinId", vf.createLiteral(binId + 3*period));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId + 3 * period));
expectedResults.add(bs);
// Verify the end results of the query match the expected results.
@@ -725,10 +715,190 @@ public class QueryIT extends RyaExportITBase {
}
+ @Test
+ public void nestedPeriodicQueryTestWithAggregationAndGroupBy() throws Exception {
+ String query = "prefix function: <http://org.apache.rya/function#> " // n
+ + "prefix time: <http://www.w3.org/2006/time#> " // n
+ + "select ?location ?total "
+ + "where { Filter(?total > 1) {"
+ + "select ?location (count(?obs) as ?total) where {" // n
+ + "Filter(function:periodic(?time, 2, .5, time:hours)) " // n
+ + "?obs <uri:hasTime> ?time. " // n
+ + "?obs <uri:hasLoc> ?location } group by ?location }}"; // n
+
+ // Create the Statements that will be loaded into Rya.
+ final ValueFactory vf = new ValueFactoryImpl();
+ final DatatypeFactory dtf = DatatypeFactory.newInstance();
+ ZonedDateTime time = ZonedDateTime.now();
+ long currentTime = time.toInstant().toEpochMilli();
+
+ ZonedDateTime zTime1 = time.minusMinutes(30);
+ String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime2 = zTime1.minusMinutes(30);
+ String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime3 = zTime2.minusMinutes(30);
+ String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime4 = zTime3.minusMinutes(30);
+ String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT);
+
+ final Collection<Statement> statements = Sets.newHashSet(
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasLoc"), vf.createLiteral("loc_1")),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasLoc"), vf.createLiteral("loc_2")),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasLoc"), vf.createLiteral("loc_3")),
+ vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
+ vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasLoc"), vf.createLiteral("loc_4")),
+ vf.createStatement(vf.createURI("urn:obs_5"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
+ vf.createStatement(vf.createURI("urn:obs_5"), vf.createURI("uri:hasLoc"), vf.createLiteral("loc_1")),
+ vf.createStatement(vf.createURI("urn:obs_6"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
+ vf.createStatement(vf.createURI("urn:obs_6"), vf.createURI("uri:hasLoc"), vf.createLiteral("loc_2")));
+
+ // Create the expected results of the SPARQL query once the PCJ has been computed.
+ final Set<BindingSet> expectedResults = new HashSet<>();
+
+ long period = 1800000;
+ long binId = (currentTime / period) * period;
+
+ MapBindingSet bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
+ bs.addBinding("location", vf.createLiteral("loc_1", XMLSchema.STRING));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId));
+ expectedResults.add(bs);
+
+ bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
+ bs.addBinding("location", vf.createLiteral("loc_2", XMLSchema.STRING));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId));
+ expectedResults.add(bs);
+
+
+ bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
+ bs.addBinding("location", vf.createLiteral("loc_2", XMLSchema.STRING));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId + period));
+ expectedResults.add(bs);
+
+ // Verify the end results of the query match the expected results.
+ runTest(query, statements, expectedResults, ExporterType.Periodic);
+ }
-
+ @Test
+ public void nestedJoinPeriodicQueryWithAggregationAndGroupBy() throws Exception {
+ String query = "prefix function: <http://org.apache.rya/function#> " // n
+ + "prefix time: <http://www.w3.org/2006/time#> " // n
+ + "select ?location ?total ?population "
+ + "where { Filter(?total > 1)"
+ + "?location <uri:hasPopulation> ?population . {"
+ + "select ?location (count(?obs) as ?total) where {" // n
+ + "Filter(function:periodic(?time, 2, .5, time:hours)) " // n
+ + "?obs <uri:hasTime> ?time. " // n
+ + "?obs <uri:hasLoc> ?location } group by ?location }}"; // n
+
+ // Create the Statements that will be loaded into Rya.
+ final ValueFactory vf = new ValueFactoryImpl();
+ final DatatypeFactory dtf = DatatypeFactory.newInstance();
+ ZonedDateTime time = ZonedDateTime.now();
+ long currentTime = time.toInstant().toEpochMilli();
+
+ ZonedDateTime zTime1 = time.minusMinutes(30);
+ String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime2 = zTime1.minusMinutes(30);
+ String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime3 = zTime2.minusMinutes(30);
+ String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
+
+ ZonedDateTime zTime4 = zTime3.minusMinutes(30);
+ String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT);
+
+ final Collection<Statement> statements = Sets.newHashSet(
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time1))),
+ vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasLoc"), vf.createURI("uri:loc_1")),
+ vf.createStatement(vf.createURI("uri:loc_1"), vf.createURI("uri:hasPopulation"), vf.createLiteral(3500)),
+ vf.createStatement(vf.createURI("uri:loc_2"), vf.createURI("uri:hasPopulation"), vf.createLiteral(8000)),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time2))),
+ vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasLoc"), vf.createURI("uri:loc_2")),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
+ vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasLoc"), vf.createURI("uri:loc_3")),
+ vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
+ vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasLoc"), vf.createURI("uri:loc_4")),
+ vf.createStatement(vf.createURI("urn:obs_5"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time4))),
+ vf.createStatement(vf.createURI("urn:obs_5"), vf.createURI("uri:hasLoc"), vf.createURI("uri:loc_1")),
+ vf.createStatement(vf.createURI("urn:obs_6"), vf.createURI("uri:hasTime"),
+ vf.createLiteral(dtf.newXMLGregorianCalendar(time3))),
+ vf.createStatement(vf.createURI("urn:obs_6"), vf.createURI("uri:hasLoc"), vf.createURI("uri:loc_2")));
+
+ // Create the expected results of the SPARQL query once the PCJ has been computed.
+ final Set<BindingSet> expectedResults = new HashSet<>();
+
+ long period = 1800000;
+ long binId = (currentTime / period) * period;
+
+ MapBindingSet bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
+ bs.addBinding("location", vf.createURI("uri:loc_1"));
+ bs.addBinding("population", vf.createLiteral("3500", XMLSchema.INTEGER));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId));
+ expectedResults.add(bs);
+
+ bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
+ bs.addBinding("location", vf.createURI("uri:loc_2"));
+ bs.addBinding("population", vf.createLiteral("8000", XMLSchema.INTEGER));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId));
+ expectedResults.add(bs);
+
- public void runTest(final String sparql, final Collection<Statement> statements, final Collection<BindingSet> expectedResults, ExporterType type ) throws Exception {
+ bs = new MapBindingSet();
+ bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
+ bs.addBinding("location", vf.createURI("uri:loc_2"));
+ bs.addBinding("population", vf.createLiteral("8000", XMLSchema.INTEGER));
+ bs.addBinding("periodicBinId", vf.createLiteral(binId + period));
+ expectedResults.add(bs);
+
+ // Verify the end results of the query match the expected results.
+ runTest(query, statements, expectedResults, ExporterType.Periodic);
+ }
+
+ @Test(expected= IllegalArgumentException.class)
+ public void nestedConstructPeriodicQueryWithAggregationAndGroupBy() throws Exception {
+ String query = "prefix function: <http://org.apache.rya/function#> " // n
+ + "prefix time: <http://www.w3.org/2006/time#> " // n
+ + "construct{?location a <uri:highObservationArea> } "
+ + "where { Filter(?total > 1)"
+ + "?location <uri:hasPopulation> ?population . {"
+ + "select ?location (count(?obs) as ?total) where {" // n
+ + "Filter(function:periodic(?time, 2, .5, time:hours)) " // n
+ + "?obs <uri:hasTime> ?time. " // n
+ + "?obs <uri:hasLoc> ?location } group by ?location }}"; // n
+
+
+ final Collection<Statement> statements = Sets.newHashSet();
+ final Set<BindingSet> expectedResults = new HashSet<>();
+
+ // Verify the end results of the query match the expected results.
+ runTest(query, statements, expectedResults, ExporterType.Periodic);
+ }
+
+ public void runTest(final String sparql, final Collection<Statement> statements, final Collection<BindingSet> expectedResults,
+ ExporterType type) throws Exception {
requireNonNull(sparql);
requireNonNull(statements);
requireNonNull(expectedResults);
@@ -754,9 +924,10 @@ public class QueryIT extends RyaExportITBase {
PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(accumuloConn, getRyaInstanceName());
String periodicId = periodicStorage.createPeriodicQuery(sparql);
try (FluoClient fluo = new FluoClientImpl(super.getFluoConfiguration())) {
- new CreatePcj().createPcj(periodicId, sparql, fluo);
+ new CreateFluoPcj().createPcj(periodicId, sparql, fluo);
}
addStatementsAndWait(statements);
+
final Set<BindingSet> results = Sets.newHashSet();
try (CloseableIterator<BindingSet> resultIter = periodicStorage.listResults(periodicId, Optional.empty())) {
while (resultIter.hasNext()) {
@@ -767,9 +938,9 @@ public class QueryIT extends RyaExportITBase {
break;
}
}
-
+
private void addStatementsAndWait(final Collection<Statement> statements) throws RepositoryException, Exception {
- // Write the data to Rya.
+ // Write the data to Rya.
final SailRepositoryConnection ryaConn = super.getRyaSailRepository().getConnection();
ryaConn.begin();
ryaConn.add(statements);
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaExportIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaExportIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaExportIT.java
index 12c69ca..15ff1b4 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaExportIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaExportIT.java
@@ -28,7 +28,7 @@ import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.api.client.FluoFactory;
import org.apache.rya.api.domain.RyaStatement;
import org.apache.rya.api.domain.RyaURI;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePcj;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
import org.apache.rya.indexing.pcj.fluo.api.InsertTriples;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage;
@@ -110,7 +110,7 @@ public class RyaExportIT extends RyaExportITBase {
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
// Tell the Fluo app to maintain the PCJ.
- new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
+ new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
// Stream the data into Fluo.
new InsertTriples().insert(fluoClient, streamedTriples, Optional.<String>absent());
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaInputIncrementalUpdateIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaInputIncrementalUpdateIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaInputIncrementalUpdateIT.java
index e6d287e..5cd3ab1 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaInputIncrementalUpdateIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaInputIncrementalUpdateIT.java
@@ -28,7 +28,7 @@ import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.api.client.FluoFactory;
import org.apache.rya.accumulo.AccumuloRyaDAO;
import org.apache.rya.indexing.external.PrecomputedJoinIndexer;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePcj;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage;
@@ -99,7 +99,7 @@ public class RyaInputIncrementalUpdateIT extends RyaExportITBase {
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
// Tell the Fluo app to maintain the PCJ.
- new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
+ new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
// Verify the end results of the query match the expected results.
super.getMiniFluo().waitForObservers();
@@ -165,7 +165,7 @@ public class RyaInputIncrementalUpdateIT extends RyaExportITBase {
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
// Tell the Fluo app to maintain the PCJ.
- new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
+ new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
super.getMiniFluo().waitForObservers();
@@ -236,7 +236,7 @@ public class RyaInputIncrementalUpdateIT extends RyaExportITBase {
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
// Tell the Fluo app to maintain the PCJ.
- new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
+ new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
super.getMiniFluo().waitForObservers();
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/StreamingTestIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/StreamingTestIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/StreamingTestIT.java
index 3f51311..e83a894 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/StreamingTestIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/StreamingTestIT.java
@@ -28,7 +28,7 @@ import org.apache.accumulo.core.client.Connector;
import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.api.client.FluoFactory;
import org.apache.log4j.Logger;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePcj;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage;
@@ -60,7 +60,7 @@ public class StreamingTestIT extends RyaExportITBase {
final String pcjId = pcjStorage.createPcj(sparql);
// Task the Fluo app with the PCJ.
- new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
+ new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
// Add Statements to the Fluo app.
log.info("Adding Join Pairs...");
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/HistoricStreamingVisibilityIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/HistoricStreamingVisibilityIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/HistoricStreamingVisibilityIT.java
index ab42e89..eab99b8 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/HistoricStreamingVisibilityIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/HistoricStreamingVisibilityIT.java
@@ -32,7 +32,7 @@ import org.apache.rya.api.RdfCloudTripleStoreConfiguration;
import org.apache.rya.api.domain.RyaStatement;
import org.apache.rya.api.resolver.RdfToRyaConversions;
import org.apache.rya.indexing.accumulo.ConfigUtils;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePcj;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage;
import org.apache.rya.pcj.fluo.test.base.RyaExportITBase;
@@ -107,7 +107,7 @@ public class HistoricStreamingVisibilityIT extends RyaExportITBase {
final String pcjId = pcjStorage.createPcj(sparql);
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
- new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
+ new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
}
// Verify the end results of the query match the expected results.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/PcjVisibilityIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/PcjVisibilityIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/PcjVisibilityIT.java
index dc2f859..2497793 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/PcjVisibilityIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/PcjVisibilityIT.java
@@ -49,7 +49,7 @@ import org.apache.rya.api.domain.RyaStatement;
import org.apache.rya.api.domain.RyaURI;
import org.apache.rya.indexing.accumulo.ConfigUtils;
import org.apache.rya.indexing.external.PrecomputedJoinIndexerConfig;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePcj;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
import org.apache.rya.indexing.pcj.fluo.api.InsertTriples;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
@@ -220,7 +220,7 @@ public class PcjVisibilityIT extends RyaExportITBase {
try( final FluoClient fluoClient = FluoFactory.newClient( super.getFluoConfiguration() )) {
// Create the PCJ in Fluo.
- new CreatePcj().withRyaIntegration(pcjId, rootStorage, fluoClient, accumuloConn, getRyaInstanceName());
+ new CreateFluoPcj().withRyaIntegration(pcjId, rootStorage, fluoClient, accumuloConn, getRyaInstanceName());
// Stream the data into Fluo.
for(final RyaStatement statement : streamedTriples.keySet()) {
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/KafkaExportITBase.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/KafkaExportITBase.java b/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/KafkaExportITBase.java
index 85da422..c828a20 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/KafkaExportITBase.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/KafkaExportITBase.java
@@ -37,6 +37,7 @@ import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.fluo.api.config.ObserverSpecification;
import org.apache.fluo.recipes.test.AccumuloExportITBase;
+import org.apache.fluo.recipes.test.FluoITHelper;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
@@ -59,6 +60,7 @@ import org.apache.rya.indexing.pcj.fluo.app.observers.AggregationObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.ConstructQueryResultObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.FilterObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.JoinObserver;
+import org.apache.rya.indexing.pcj.fluo.app.observers.ProjectionObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.QueryResultObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.StatementPatternObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.TripleObserver;
@@ -72,7 +74,6 @@ import org.openrdf.model.Statement;
import org.openrdf.repository.sail.SailRepositoryConnection;
import org.openrdf.sail.Sail;
-
import kafka.admin.AdminUtils;
import kafka.admin.RackAwareMode;
import kafka.server.KafkaConfig;
@@ -117,6 +118,7 @@ public class KafkaExportITBase extends AccumuloExportITBase {
observers.add(new ObserverSpecification(JoinObserver.class.getName()));
observers.add(new ObserverSpecification(FilterObserver.class.getName()));
observers.add(new ObserverSpecification(AggregationObserver.class.getName()));
+ observers.add(new ObserverSpecification(ProjectionObserver.class.getName()));
// Configure the export observer to export new PCJ results to the mini
// accumulo cluster.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/RyaExportITBase.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/RyaExportITBase.java b/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/RyaExportITBase.java
index 6feadff..9c5732f 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/RyaExportITBase.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/RyaExportITBase.java
@@ -32,6 +32,7 @@ import org.apache.rya.indexing.pcj.fluo.app.observers.AggregationObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.FilterObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.JoinObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.PeriodicQueryObserver;
+import org.apache.rya.indexing.pcj.fluo.app.observers.ProjectionObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.QueryResultObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.StatementPatternObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.TripleObserver;
@@ -59,6 +60,7 @@ public class RyaExportITBase extends FluoITBase {
observers.add(new ObserverSpecification(FilterObserver.class.getName()));
observers.add(new ObserverSpecification(AggregationObserver.class.getName()));
observers.add(new ObserverSpecification(PeriodicQueryObserver.class.getName()));
+ observers.add(new ObserverSpecification(ProjectionObserver.class.getName()));
// Configure the export observer to export new PCJ results to the mini accumulo cluster.
final HashMap<String, String> exportParams = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationProviderIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationProviderIT.java b/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationProviderIT.java
index 1902248..4d1bc75 100644
--- a/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationProviderIT.java
+++ b/extras/rya.periodic.service/periodic.service.integration.tests/src/test/java/org/apache/rya/periodic/notification/application/PeriodicNotificationProviderIT.java
@@ -25,7 +25,8 @@ import java.util.concurrent.TimeUnit;
import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.core.client.FluoClientImpl;
import org.apache.fluo.recipes.test.AccumuloExportITBase;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePcj;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
+import org.apache.rya.indexing.pcj.fluo.app.util.FluoQueryUtils;
import org.apache.rya.periodic.notification.coordinator.PeriodicNotificationCoordinatorExecutor;
import org.apache.rya.periodic.notification.notification.TimestampedNotification;
import org.apache.rya.periodic.notification.recovery.PeriodicNotificationProvider;
@@ -49,11 +50,11 @@ public class PeriodicNotificationProviderIT extends AccumuloExportITBase {
BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
PeriodicNotificationCoordinatorExecutor coord = new PeriodicNotificationCoordinatorExecutor(2, notifications);
PeriodicNotificationProvider provider = new PeriodicNotificationProvider();
- CreatePcj pcj = new CreatePcj();
+ CreateFluoPcj pcj = new CreateFluoPcj();
String id = null;
try(FluoClient fluo = new FluoClientImpl(getFluoConfiguration())) {
- id = pcj.createPcj(sparql, fluo);
+ id = pcj.createPcj(sparql, fluo).getQueryId();
provider.processRegisteredNotifications(coord, fluo.newSnapshot());
}
@@ -61,7 +62,7 @@ public class PeriodicNotificationProviderIT extends AccumuloExportITBase {
Assert.assertEquals(5000, notification.getInitialDelay());
Assert.assertEquals(15000, notification.getPeriod());
Assert.assertEquals(TimeUnit.MILLISECONDS, notification.getTimeUnit());
- Assert.assertEquals(id, notification.getId());
+ Assert.assertEquals(FluoQueryUtils.convertFluoQueryIdToPcjId(id), notification.getId());
}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/api/CreatePeriodicQuery.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/api/CreatePeriodicQuery.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/api/CreatePeriodicQuery.java
index 7f71b52..6aade52 100644
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/api/CreatePeriodicQuery.java
+++ b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/api/CreatePeriodicQuery.java
@@ -21,8 +21,9 @@ package org.apache.rya.periodic.notification.api;
import java.util.Optional;
import org.apache.fluo.api.client.FluoClient;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePcj;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
import org.apache.rya.indexing.pcj.fluo.app.query.PeriodicQueryNode;
+import org.apache.rya.indexing.pcj.fluo.app.util.FluoQueryUtils;
import org.apache.rya.indexing.pcj.fluo.app.util.PeriodicQueryUtil;
import org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage;
import org.apache.rya.indexing.pcj.storage.PeriodicQueryStorageException;
@@ -31,6 +32,8 @@ import org.apache.rya.periodic.notification.notification.PeriodicNotification;
import org.openrdf.query.MalformedQueryException;
import org.openrdf.query.algebra.evaluation.function.Function;
+import com.google.common.base.Preconditions;
+
/**
* Object that creates a Periodic Query. A Periodic Query is any query
* requesting periodic updates about events that occurred within a given
@@ -79,8 +82,9 @@ public class CreatePeriodicQuery {
Optional<PeriodicQueryNode> optNode = PeriodicQueryUtil.getPeriodicNode(sparql);
if(optNode.isPresent()) {
PeriodicQueryNode periodicNode = optNode.get();
- CreatePcj createPcj = new CreatePcj();
- String queryId = createPcj.createPcj(sparql, fluoClient);
+ CreateFluoPcj createPcj = new CreateFluoPcj();
+ String queryId = createPcj.createPcj(sparql, fluoClient).getQueryId();
+ queryId = FluoQueryUtils.convertFluoQueryIdToPcjId(queryId);
periodicStorage.createPeriodicQuery(queryId, sparql);
PeriodicNotification notification = PeriodicNotification.builder().id(queryId).period(periodicNode.getPeriod())
.timeUnit(periodicNode.getUnit()).build();
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/recovery/PeriodicNotificationProvider.java
----------------------------------------------------------------------
diff --git a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/recovery/PeriodicNotificationProvider.java b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/recovery/PeriodicNotificationProvider.java
index 8e8b1a2..27e06f0 100644
--- a/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/recovery/PeriodicNotificationProvider.java
+++ b/extras/rya.periodic.service/periodic.service.notification/src/main/java/org/apache/rya/periodic/notification/recovery/PeriodicNotificationProvider.java
@@ -126,11 +126,14 @@ public class PeriodicNotificationProvider {
id = getQueryIdFromPeriodicId(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.AGGREGATION_PARENT_NODE_ID).toString());
break;
case CONSTRUCT:
- id = sx.get(Bytes.of(nodeId), FluoQueryColumns.CONSTRUCT_NODE_ID).toString();
- id = id.split(IncrementalUpdateConstants.CONSTRUCT_PREFIX)[1];
+ id = getQueryIdFromPeriodicId(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.CONSTRUCT_PARENT_NODE_ID).toString());
+ break;
+ case PROJECTION:
+ id = getQueryIdFromPeriodicId(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.PROJECTION_PARENT_NODE_ID).toString());
break;
default:
- throw new RuntimeException("Invalid NodeType.");
+ throw new IllegalArgumentException("Invalid node type");
+
}
return id;
}
[2/5] incubator-rya git commit: RYA-282-Nested-Query. Closes #192.
Posted by ca...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/util/QueryReportRenderer.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/util/QueryReportRenderer.java b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/util/QueryReportRenderer.java
index 7a73b41..f44db6c 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/util/QueryReportRenderer.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/util/QueryReportRenderer.java
@@ -25,10 +25,12 @@ import edu.umd.cs.findbugs.annotations.NonNull;
import org.apache.commons.lang3.StringUtils;
import org.apache.rya.indexing.pcj.fluo.api.GetQueryReport.QueryReport;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.QueryType;
import org.apache.rya.indexing.pcj.fluo.app.query.ConstructQueryMetadata;
import org.apache.rya.indexing.pcj.fluo.app.query.FilterMetadata;
import org.apache.rya.indexing.pcj.fluo.app.query.FluoQuery;
import org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.ProjectionMetadata;
import org.apache.rya.indexing.pcj.fluo.app.query.QueryMetadata;
import org.apache.rya.indexing.pcj.fluo.app.query.StatementPatternMetadata;
import org.apache.rya.indexing.pcj.fluo.client.util.Report.ReportItem;
@@ -56,26 +58,41 @@ public class QueryReportRenderer {
final FluoQuery metadata = queryReport.getFluoQuery();
- switch (metadata.getQueryType()) {
- case Projection:
- final QueryMetadata queryMetadata = metadata.getQueryMetadata().get();
- builder.appendItem(new ReportItem("QUERY NODE"));
- builder.appendItem(new ReportItem("Node ID", queryMetadata.getNodeId()));
- builder.appendItem(new ReportItem("Variable Order", queryMetadata.getVariableOrder().toString()));
- builder.appendItem(new ReportItem("SPARQL", prettyFormatSparql(queryMetadata.getSparql())));
- builder.appendItem(new ReportItem("Child Node ID", queryMetadata.getChildNodeId()));
- builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(queryMetadata.getNodeId())));
- break;
- case Construct:
+ QueryMetadata queryMetadata = metadata.getQueryMetadata();
+ builder.appendItem( new ReportItem("") );
+
+ builder.appendItem(new ReportItem("QUERY NODE"));
+ builder.appendItem(new ReportItem("Node ID", queryMetadata.getNodeId()));
+ builder.appendItem(new ReportItem("Variable Order", queryMetadata.getVariableOrder().toString()));
+ builder.appendItem( new ReportItem("SPARQL", queryMetadata.getSparql()) );
+ builder.appendItem(new ReportItem("Child Node ID", queryMetadata.getChildNodeId()));
+ builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(queryMetadata.getNodeId())));
+
+
+
+ if (metadata.getQueryType() == QueryType.Construct) {
+ builder.appendItem( new ReportItem("") );
+
final ConstructQueryMetadata constructMetadata = metadata.getConstructQueryMetadata().get();
builder.appendItem(new ReportItem("CONSTRUCT QUERY NODE"));
builder.appendItem(new ReportItem("Node ID", constructMetadata.getNodeId()));
builder.appendItem(new ReportItem("Variable Order", constructMetadata.getVariableOrder().toString()));
- builder.appendItem(new ReportItem("SPARQL", prettyFormatSparql(constructMetadata.getSparql())));
+ builder.appendItem( new ReportItem("Parent Node ID", constructMetadata.getParentNodeId()) );
builder.appendItem(new ReportItem("Child Node ID", constructMetadata.getChildNodeId()));
builder.appendItem(new ReportItem("Construct Graph", constructMetadata.getConstructGraph().toString()));
builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(constructMetadata.getNodeId())));
}
+
+ for (ProjectionMetadata projectionMetadata : metadata.getProjectionMetadata()) {
+ builder.appendItem( new ReportItem("") );
+
+ builder.appendItem(new ReportItem("PROJECTION NODE"));
+ builder.appendItem(new ReportItem("Node ID", projectionMetadata.getNodeId()));
+ builder.appendItem(new ReportItem("Variable Order", projectionMetadata.getVariableOrder().toString()));
+ builder.appendItem( new ReportItem("Parent Node ID", projectionMetadata.getParentNodeId()) );
+ builder.appendItem(new ReportItem("Child Node ID", projectionMetadata.getChildNodeId()));
+ builder.appendItem(new ReportItem("Count", "" + queryReport.getCount(projectionMetadata.getNodeId())));
+ }
for(final FilterMetadata filterMetadata : metadata.getFilterMetadata()) {
builder.appendItem( new ReportItem("") );
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/FluoAndHistoricPcjsDemo.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/FluoAndHistoricPcjsDemo.java b/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/FluoAndHistoricPcjsDemo.java
index c8dc737..f25b573 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/FluoAndHistoricPcjsDemo.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/FluoAndHistoricPcjsDemo.java
@@ -33,7 +33,7 @@ import org.apache.rya.api.domain.RyaType;
import org.apache.rya.api.domain.RyaURI;
import org.apache.rya.api.persist.RyaDAOException;
import org.apache.rya.api.resolver.RyaToRdfConversions;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePcj;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
import org.apache.rya.indexing.pcj.fluo.api.InsertTriples;
import org.apache.rya.indexing.pcj.storage.PcjException;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
@@ -179,7 +179,7 @@ public class FluoAndHistoricPcjsDemo implements Demo {
pcjId = pcjStorage.createPcj(sparql);
// Tell the Fluo app to maintain it.
- new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, ryaTablePrefix);
+ new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, ryaTablePrefix);
} catch (MalformedQueryException | PcjException | RyaDAOException e) {
throw new DemoExecutionException("Error while using Fluo to compute and export historic matches, so the demo can not continue. Exiting.", e);
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ConstructGraphTestUtils.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ConstructGraphTestUtils.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ConstructGraphTestUtils.java
index 124569b..bbdcfec 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ConstructGraphTestUtils.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ConstructGraphTestUtils.java
@@ -30,6 +30,7 @@ import org.junit.Assert;
import org.openrdf.model.Statement;
import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
public class ConstructGraphTestUtils {
@@ -47,8 +48,18 @@ public class ConstructGraphTestUtils {
public static void subGraphsEqualIgnoresBlankNode(Set<RyaSubGraph> subgraph1, Set<RyaSubGraph> subgraph2) {
Map<Integer, RyaSubGraph> subGraphMap = new HashMap<>();
- subgraph1.forEach(x->subGraphMap.put(getKey(x), x));
- subgraph2.forEach(x->ryaStatementsEqualIgnoresBlankNode(x.getStatements(), subGraphMap.get(getKey(x)).getStatements()));
+ for(RyaSubGraph subgraph: subgraph1) {
+ int key = getKey(subgraph);
+ subGraphMap.put(key, subgraph);
+ }
+
+ for(RyaSubGraph subgraph: subgraph2) {
+ int key = getKey(subgraph);
+ RyaSubGraph sub = subGraphMap.get(key);
+ Preconditions.checkNotNull(sub);
+ Set<RyaStatement> statements = sub.getStatements();
+ ryaStatementsEqualIgnoresBlankNode(subgraph.getStatements(), statements);
+ }
}
private static int getKey(RyaSubGraph subgraph) {
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/GetPcjMetadataIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/GetPcjMetadataIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/GetPcjMetadataIT.java
index d5c0e5f..263a19e 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/GetPcjMetadataIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/GetPcjMetadataIT.java
@@ -68,7 +68,7 @@ public class GetPcjMetadataIT extends RyaExportITBase {
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
// Tell the Fluo app to maintain the PCJ.
- new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
+ new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
// Fetch the PCJ's Metadata through the GetPcjMetadata interactor.
final String queryId = new ListQueryIds().listQueryIds(fluoClient).get(0);
@@ -95,7 +95,7 @@ public class GetPcjMetadataIT extends RyaExportITBase {
"?x <http://worksAt> <http://Chipotle>." +
"}";
final String q1PcjId = pcjStorage.createPcj(q1Sparql);
- final CreatePcj createPcj = new CreatePcj();
+ final CreateFluoPcj createPcj = new CreateFluoPcj();
createPcj.withRyaIntegration(q1PcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
final String q2Sparql =
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/GetQueryReportIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/GetQueryReportIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/GetQueryReportIT.java
index 965a7b9..c0f0f16 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/GetQueryReportIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/GetQueryReportIT.java
@@ -85,7 +85,7 @@ public class GetQueryReportIT extends RyaExportITBase {
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
// Tell the Fluo app to maintain the PCJ.
- new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
+ new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
// Stream the data into Fluo.
new InsertTriples().insert(fluoClient, streamedTriples, Optional.<String>absent());
@@ -106,7 +106,7 @@ public class GetQueryReportIT extends RyaExportITBase {
final FluoQuery fluoQuery = report.getFluoQuery();
- final String queryNodeId = fluoQuery.getQueryMetadata().get().getNodeId();
+ final String queryNodeId = fluoQuery.getQueryMetadata().getNodeId();
expectedCounts.put(queryNodeId, BigInteger.valueOf(8));
final String filterNodeId = fluoQuery.getFilterMetadata().iterator().next().getNodeId();
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAOIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAOIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAOIT.java
index d403404..315dddb 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAOIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAOIT.java
@@ -20,6 +20,8 @@ package org.apache.rya.indexing.pcj.fluo.app.query;
import static org.junit.Assert.assertEquals;
+import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -28,11 +30,12 @@ import org.apache.fluo.api.client.FluoFactory;
import org.apache.fluo.api.client.Snapshot;
import org.apache.fluo.api.client.Transaction;
import org.apache.rya.indexing.pcj.fluo.app.ConstructGraph;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.ExportStrategy;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.QueryType;
+import org.apache.rya.indexing.pcj.fluo.app.NodeType;
import org.apache.rya.indexing.pcj.fluo.app.query.AggregationMetadata.AggregationElement;
import org.apache.rya.indexing.pcj.fluo.app.query.AggregationMetadata.AggregationType;
-import org.apache.rya.indexing.pcj.fluo.app.query.FluoQuery.QueryType;
import org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata.JoinType;
-import org.apache.rya.indexing.pcj.fluo.app.query.SparqlFluoQueryBuilder.NodeIds;
import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
import org.apache.rya.pcj.fluo.test.base.RyaExportITBase;
import org.junit.Test;
@@ -113,7 +116,7 @@ public class FluoQueryMetadataDAOIT extends RyaExportITBase {
// Create the object that will be serialized.
final JoinMetadata.Builder builder = JoinMetadata.builder("nodeId");
- builder.setVariableOrder(new VariableOrder("g;y;s"));
+ builder.setVarOrder(new VariableOrder("g;y;s"));
builder.setJoinType(JoinType.NATURAL_JOIN);
builder.setParentNodeId("parentNodeId");
builder.setLeftChildNodeId("leftChildNodeId");
@@ -143,10 +146,13 @@ public class FluoQueryMetadataDAOIT extends RyaExportITBase {
final FluoQueryMetadataDAO dao = new FluoQueryMetadataDAO();
// Create the object that will be serialized.
- final QueryMetadata.Builder builder = QueryMetadata.builder("nodeId");
- builder.setVariableOrder(new VariableOrder("y;s;d"));
+ String queryId = NodeType.generateNewFluoIdForType(NodeType.QUERY);
+ final QueryMetadata.Builder builder = QueryMetadata.builder(queryId);
+ builder.setQueryType(QueryType.Projection);
+ builder.setVarOrder(new VariableOrder("y;s;d"));
builder.setSparql("sparql string");
builder.setChildNodeId("childNodeId");
+ builder.setExportStrategies(new HashSet<>(Arrays.asList(ExportStrategy.Kafka)));
final QueryMetadata originalMetadata = builder.build();
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
@@ -159,7 +165,37 @@ public class FluoQueryMetadataDAOIT extends RyaExportITBase {
// Read it from the Fluo table.
QueryMetadata storedMetdata = null;
try(Snapshot sx = fluoClient.newSnapshot()) {
- storedMetdata = dao.readQueryMetadata(sx, "nodeId");
+ storedMetdata = dao.readQueryMetadata(sx, queryId);
+ }
+
+ // Ensure the deserialized object is the same as the serialized one.
+ assertEquals(originalMetadata, storedMetdata);
+ }
+ }
+
+ @Test
+ public void projectionMetadataTest() {
+ final FluoQueryMetadataDAO dao = new FluoQueryMetadataDAO();
+
+ // Create the object that will be serialized.
+ final ProjectionMetadata.Builder builder = ProjectionMetadata.builder("nodeId");
+ builder.setVarOrder(new VariableOrder("y;s;d"));
+ builder.setProjectedVars(new VariableOrder("x;y;z"));
+ builder.setChildNodeId("childNodeId");
+ builder.setParentNodeId("parentNodeId");
+ final ProjectionMetadata originalMetadata = builder.build();
+
+ try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
+ // Write it to the Fluo table.
+ try(Transaction tx = fluoClient.newTransaction()) {
+ dao.write(tx, originalMetadata);
+ tx.commit();
+ }
+
+ // Read it from the Fluo table.
+ ProjectionMetadata storedMetdata = null;
+ try(Snapshot sx = fluoClient.newSnapshot()) {
+ storedMetdata = dao.readProjectionMetadata(sx, "nodeId");
}
// Ensure the deserialized object is the same as the serialized one.
@@ -180,8 +216,9 @@ public class FluoQueryMetadataDAOIT extends RyaExportITBase {
// Create the object that will be serialized.
final ConstructQueryMetadata.Builder builder = ConstructQueryMetadata.builder();
builder.setNodeId("nodeId");
- builder.setSparql(query);
builder.setChildNodeId("childNodeId");
+ builder.setParentNodeId("parentNodeId");
+ builder.setVarOrder(new VariableOrder("a;b;c"));
builder.setConstructGraph(new ConstructGraph(patterns));
final ConstructQueryMetadata originalMetadata = builder.build();
@@ -209,7 +246,7 @@ public class FluoQueryMetadataDAOIT extends RyaExportITBase {
// Create the object that will be serialized.
final AggregationMetadata originalMetadata = AggregationMetadata.builder("nodeId")
- .setVariableOrder(new VariableOrder("totalCount"))
+ .setVarOrder(new VariableOrder("totalCount"))
.setParentNodeId("parentNodeId")
.setChildNodeId("childNodeId")
.setGroupByVariableOrder(new VariableOrder("a", "b", "c"))
@@ -240,7 +277,7 @@ public class FluoQueryMetadataDAOIT extends RyaExportITBase {
// Create the object that will be serialized.
final AggregationMetadata originalMetadata = AggregationMetadata.builder("nodeId")
- .setVariableOrder(new VariableOrder("totalCount"))
+ .setVarOrder(new VariableOrder("totalCount"))
.setParentNodeId("parentNodeId")
.setChildNodeId("childNodeId")
.addAggregation(new AggregationElement(AggregationType.COUNT, "count", "totalCount"))
@@ -315,8 +352,10 @@ public class FluoQueryMetadataDAOIT extends RyaExportITBase {
"?worker <http://worksAt> <http://Chipotle>. " +
"}";
- final ParsedQuery query = new SPARQLParser().parseQuery(sparql, null);
- final FluoQuery originalQuery = new SparqlFluoQueryBuilder().make(query, new NodeIds());
+ SparqlFluoQueryBuilder builder = new SparqlFluoQueryBuilder();
+ builder.setSparql(sparql);
+ builder.setFluoQueryId(NodeType.generateNewFluoIdForType(NodeType.QUERY));
+ final FluoQuery originalQuery = builder.build();
assertEquals(QueryType.Projection, originalQuery.getQueryType());
assertEquals(false, originalQuery.getConstructQueryMetadata().isPresent());
@@ -331,7 +370,7 @@ public class FluoQueryMetadataDAOIT extends RyaExportITBase {
// Read it from the Fluo table.
FluoQuery storedQuery = null;
try(Snapshot sx = fluoClient.newSnapshot()) {
- storedQuery = dao.readFluoQuery(sx, originalQuery.getQueryMetadata().get().getNodeId());
+ storedQuery = dao.readFluoQuery(sx, originalQuery.getQueryMetadata().getNodeId());
}
// Ensure the deserialized object is the same as the serialized one.
@@ -354,11 +393,102 @@ public class FluoQueryMetadataDAOIT extends RyaExportITBase {
"?worker <http://worksAt> <http://Chipotle>. " +
"}";
- final ParsedQuery query = new SPARQLParser().parseQuery(sparql, null);
- final FluoQuery originalQuery = new SparqlFluoQueryBuilder().make(query, new NodeIds());
+ SparqlFluoQueryBuilder builder = new SparqlFluoQueryBuilder();
+ builder.setSparql(sparql);
+ builder.setFluoQueryId(NodeType.generateNewFluoIdForType(NodeType.QUERY));
+ final FluoQuery originalQuery = builder.build();
+
+ assertEquals(QueryType.Construct, originalQuery.getQueryType());
+ assertEquals(true, originalQuery.getConstructQueryMetadata().isPresent());
+
+ try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
+ // Write it to the Fluo table.
+ try(Transaction tx = fluoClient.newTransaction()) {
+ dao.write(tx, originalQuery);
+ tx.commit();
+ }
+
+ // Read it from the Fluo table.
+ FluoQuery storedQuery = null;
+ try(Snapshot sx = fluoClient.newSnapshot()) {
+ storedQuery = dao.readFluoQuery(sx, originalQuery.getQueryMetadata().getNodeId());
+ }
+
+ // Ensure the deserialized object is the same as the serialized one.
+ assertEquals(originalQuery, storedQuery);
+ }
+ }
+
+
+ @Test
+ public void fluoNestedQueryTest() throws MalformedQueryException {
+ final FluoQueryMetadataDAO dao = new FluoQueryMetadataDAO();
+
+ // Create the object that will be serialized.
+ final String sparql =
+ "SELECT ?id ?type ?location ?averagePrice ?vendor {" +
+ "FILTER(?averagePrice > 4) " +
+ "?type <urn:purchasedFrom> ?vendor ." +
+ "{SELECT ?type ?location (avg(?price) as ?averagePrice) {" +
+ "?id <urn:type> ?type . " +
+ "?id <urn:location> ?location ." +
+ "?id <urn:price> ?price ." +
+ "} " +
+ "GROUP BY ?type ?location }}";
+
+
+ SparqlFluoQueryBuilder builder = new SparqlFluoQueryBuilder();
+ builder.setSparql(sparql);
+ builder.setFluoQueryId(NodeType.generateNewFluoIdForType(NodeType.QUERY));
+ final FluoQuery originalQuery = builder.build();
+
+ assertEquals(QueryType.Projection, originalQuery.getQueryType());
+
+ try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
+ // Write it to the Fluo table.
+ try(Transaction tx = fluoClient.newTransaction()) {
+ dao.write(tx, originalQuery);
+ tx.commit();
+ }
+
+ // Read it from the Fluo table.
+ FluoQuery storedQuery = null;
+ try(Snapshot sx = fluoClient.newSnapshot()) {
+ storedQuery = dao.readFluoQuery(sx, originalQuery.getQueryMetadata().getNodeId());
+ }
+
+ // Ensure the deserialized object is the same as the serialized one.
+ assertEquals(originalQuery, storedQuery);
+ }
+ }
+
+ @Test
+ public void fluoNestedConstructQueryTest() throws MalformedQueryException {
+ final FluoQueryMetadataDAO dao = new FluoQueryMetadataDAO();
+
+ // Create the object that will be serialized.
+ final String sparql = "CONSTRUCT { "
+ + "_:b a <urn:highSpeedTrafficArea> . "
+ + "_:b <urn:hasCount> ?obsCount . "
+ + "_:b <urn:hasLocation> ?location ."
+ + "_:b <urn:hasAverageVelocity> ?avgVelocity ."
+ + "} WHERE { "
+ + "FILTER(?obsCount > 1) "
+ + "{ "
+ + "SELECT ?location (count(?obs) AS ?obsCount) (avg(?velocity) AS ?avgVelocity) "
+ + "WHERE { "
+ + "FILTER(?velocity > 75) "
+ + "?obs <urn:hasVelocity> ?velocity. "
+ + "?obs <urn:hasLocation> ?location. "
+ + "}GROUP BY ?location }}";
+
+
+ SparqlFluoQueryBuilder builder = new SparqlFluoQueryBuilder();
+ builder.setSparql(sparql);
+ builder.setFluoQueryId(NodeType.generateNewFluoIdForType(NodeType.QUERY));
+ final FluoQuery originalQuery = builder.build();
assertEquals(QueryType.Construct, originalQuery.getQueryType());
- assertEquals(false, originalQuery.getQueryMetadata().isPresent());
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
// Write it to the Fluo table.
@@ -370,11 +500,12 @@ public class FluoQueryMetadataDAOIT extends RyaExportITBase {
// Read it from the Fluo table.
FluoQuery storedQuery = null;
try(Snapshot sx = fluoClient.newSnapshot()) {
- storedQuery = dao.readFluoQuery(sx, originalQuery.getConstructQueryMetadata().get().getNodeId());
+ storedQuery = dao.readFluoQuery(sx, originalQuery.getQueryMetadata().getNodeId());
}
// Ensure the deserialized object is the same as the serialized one.
assertEquals(originalQuery, storedQuery);
}
}
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/BatchDeleteIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/BatchDeleteIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/BatchDeleteIT.java
index 0cd7cfb..1707308 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/BatchDeleteIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/BatchDeleteIT.java
@@ -41,7 +41,7 @@ import org.apache.fluo.core.client.FluoClientImpl;
import org.apache.log4j.Logger;
import org.apache.rya.api.domain.RyaStatement;
import org.apache.rya.api.domain.RyaURI;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePcj;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
import org.apache.rya.indexing.pcj.fluo.api.InsertTriples;
import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
import org.apache.rya.indexing.pcj.fluo.app.JoinResultUpdater.Side;
@@ -91,7 +91,7 @@ public class BatchDeleteIT extends RyaExportITBase {
final String pcjId = pcjStorage.createPcj(sparql);
// Tell the Fluo app to maintain the PCJ.
- String queryId = new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, getAccumuloConnector(), getRyaInstanceName());
+ String queryId = new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, getAccumuloConnector(), getRyaInstanceName());
List<String> ids = getNodeIdStrings(fluoClient, queryId);
List<String> prefixes = Arrays.asList("urn:subject_1", "urn:object", "urn:subject_1", "urn:subject_1");
@@ -130,7 +130,7 @@ public class BatchDeleteIT extends RyaExportITBase {
final String pcjId = pcjStorage.createPcj(sparql);
// Tell the Fluo app to maintain the PCJ.
- String queryId = new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, getAccumuloConnector(), getRyaInstanceName());
+ String queryId = new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, getAccumuloConnector(), getRyaInstanceName());
List<String> ids = getNodeIdStrings(fluoClient, queryId);
String joinId = ids.get(1);
@@ -176,7 +176,7 @@ public class BatchDeleteIT extends RyaExportITBase {
final String pcjId = pcjStorage.createPcj(sparql);
// Tell the Fluo app to maintain the PCJ.
- String queryId = new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, getAccumuloConnector(), getRyaInstanceName());
+ String queryId = new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, getAccumuloConnector(), getRyaInstanceName());
List<String> ids = getNodeIdStrings(fluoClient, queryId);
String joinId = ids.get(1);
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/CreateDeleteIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/CreateDeleteIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/CreateDeleteIT.java
index 0f2d892..7c4caa4 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/CreateDeleteIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/CreateDeleteIT.java
@@ -35,7 +35,7 @@ import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.Span;
import org.apache.rya.api.client.RyaClient;
import org.apache.rya.api.client.accumulo.AccumuloRyaClientFactory;
-import org.apache.rya.indexing.pcj.fluo.api.DeletePcj;
+import org.apache.rya.indexing.pcj.fluo.api.DeleteFluoPcj;
import org.apache.rya.pcj.fluo.test.base.RyaExportITBase;
import org.junit.Test;
import org.openrdf.model.Statement;
@@ -79,10 +79,10 @@ public class CreateDeleteIT extends RyaExportITBase {
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
// Ensure the data was loaded.
final List<Bytes> rows = getFluoTableEntries(fluoClient);
- assertEquals(17, rows.size());
+ assertEquals(20, rows.size());
// Delete the PCJ from the Fluo application.
- new DeletePcj(1).deletePcj(fluoClient, pcjId);
+ new DeleteFluoPcj(1).deletePcj(fluoClient, pcjId);
// Ensure all data related to the query has been removed.
final List<Bytes> empty_rows = getFluoTableEntries(fluoClient);
@@ -111,10 +111,10 @@ public class CreateDeleteIT extends RyaExportITBase {
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
// Ensure the data was loaded.
final List<Bytes> rows = getFluoTableEntries(fluoClient);
- assertEquals(10, rows.size());
+ assertEquals(12, rows.size());
// Delete the PCJ from the Fluo application.
- new DeletePcj(1).deletePcj(fluoClient, pcjId);
+ new DeleteFluoPcj(1).deletePcj(fluoClient, pcjId);
// Ensure all data related to the query has been removed.
final List<Bytes> empty_rows = getFluoTableEntries(fluoClient);
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/InputIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/InputIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/InputIT.java
index f330825..d623043 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/InputIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/InputIT.java
@@ -29,7 +29,7 @@ import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.api.client.FluoFactory;
import org.apache.rya.api.domain.RyaStatement;
import org.apache.rya.api.domain.RyaURI;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePcj;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
import org.apache.rya.indexing.pcj.fluo.api.InsertTriples;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.CloseableIterator;
@@ -102,7 +102,7 @@ public class InputIT extends RyaExportITBase {
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
// Tell the Fluo app to maintain the PCJ.
- new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
+ new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
// Verify the end results of the query match the expected results.
super.getMiniFluo().waitForObservers();
@@ -162,7 +162,7 @@ public class InputIT extends RyaExportITBase {
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
// Tell the Fluo app to maintain the PCJ.
- new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
+ new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
// Ensure the query has no results yet.
super.getMiniFluo().waitForObservers();
@@ -228,7 +228,7 @@ public class InputIT extends RyaExportITBase {
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
// Tell the Fluo app to maintain the PCJ.
- new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
+ new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
// Ensure Alice is a match.
super.getMiniFluo().waitForObservers();
@@ -317,7 +317,7 @@ public class InputIT extends RyaExportITBase {
try(FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
// Tell the Fluo app to maintain the PCJ.
- new CreatePcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
+ new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
// Ensure Alice is a match.
super.getMiniFluo().waitForObservers();
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/KafkaExportIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/KafkaExportIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/KafkaExportIT.java
index ab7610d..3ee07a7 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/KafkaExportIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/KafkaExportIT.java
@@ -29,6 +29,9 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.core.client.FluoClientImpl;
+import org.apache.fluo.recipes.test.FluoITHelper;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
@@ -244,6 +247,10 @@ public class KafkaExportIT extends KafkaExportITBase {
// Create the PCJ in Fluo and load the statements into Rya.
final String pcjId = loadData(sparql, statements);
+
+ try(FluoClient fluo = new FluoClientImpl(super.getFluoConfiguration())) {
+ FluoITHelper.printFluoTable(fluo);
+ }
// Create the expected results of the SPARQL query once the PCJ has been computed.
final MapBindingSet expectedResult = new MapBindingSet();
@@ -350,7 +357,7 @@ public class KafkaExportIT extends KafkaExportITBase {
}
@Test
- public void groupByManyBindings_avaerages() throws Exception {
+ public void groupByManyBindings_averages() throws Exception {
// A query that groups what is aggregated by two of the keys.
final String sparql =
"SELECT ?type ?location (avg(?price) as ?averagePrice) {" +
@@ -425,6 +432,160 @@ public class KafkaExportIT extends KafkaExportITBase {
assertEquals(expectedResults, results);
}
+
+ @Test
+ public void nestedGroupByManyBindings_averages() throws Exception {
+ // A query that groups what is aggregated by two of the keys.
+ final String sparql =
+ "SELECT ?type ?location ?averagePrice {" +
+ "FILTER(?averagePrice > 4) " +
+ "{SELECT ?type ?location (avg(?price) as ?averagePrice) {" +
+ "?id <urn:type> ?type . " +
+ "?id <urn:location> ?location ." +
+ "?id <urn:price> ?price ." +
+ "} " +
+ "GROUP BY ?type ?location }}";
+
+ // Create the Statements that will be loaded into Rya.
+ final ValueFactory vf = new ValueFactoryImpl();
+ final Collection<Statement> statements = Sets.newHashSet(
+ // American items that will be averaged.
+ vf.createStatement(vf.createURI("urn:1"), vf.createURI("urn:type"), vf.createLiteral("apple")),
+ vf.createStatement(vf.createURI("urn:1"), vf.createURI("urn:location"), vf.createLiteral("USA")),
+ vf.createStatement(vf.createURI("urn:1"), vf.createURI("urn:price"), vf.createLiteral(2.50)),
+
+ vf.createStatement(vf.createURI("urn:2"), vf.createURI("urn:type"), vf.createLiteral("cheese")),
+ vf.createStatement(vf.createURI("urn:2"), vf.createURI("urn:location"), vf.createLiteral("USA")),
+ vf.createStatement(vf.createURI("urn:2"), vf.createURI("urn:price"), vf.createLiteral(4.25)),
+
+ vf.createStatement(vf.createURI("urn:3"), vf.createURI("urn:type"), vf.createLiteral("cheese")),
+ vf.createStatement(vf.createURI("urn:3"), vf.createURI("urn:location"), vf.createLiteral("USA")),
+ vf.createStatement(vf.createURI("urn:3"), vf.createURI("urn:price"), vf.createLiteral(5.25)),
+
+ // French items that will be averaged.
+ vf.createStatement(vf.createURI("urn:4"), vf.createURI("urn:type"), vf.createLiteral("cheese")),
+ vf.createStatement(vf.createURI("urn:4"), vf.createURI("urn:location"), vf.createLiteral("France")),
+ vf.createStatement(vf.createURI("urn:4"), vf.createURI("urn:price"), vf.createLiteral(8.5)),
+
+ vf.createStatement(vf.createURI("urn:5"), vf.createURI("urn:type"), vf.createLiteral("cigarettes")),
+ vf.createStatement(vf.createURI("urn:5"), vf.createURI("urn:location"), vf.createLiteral("France")),
+ vf.createStatement(vf.createURI("urn:5"), vf.createURI("urn:price"), vf.createLiteral(3.99)),
+
+ vf.createStatement(vf.createURI("urn:6"), vf.createURI("urn:type"), vf.createLiteral("cigarettes")),
+ vf.createStatement(vf.createURI("urn:6"), vf.createURI("urn:location"), vf.createLiteral("France")),
+ vf.createStatement(vf.createURI("urn:6"), vf.createURI("urn:price"), vf.createLiteral(4.99)));
+
+ // Create the PCJ in Fluo and load the statements into Rya.
+ final String pcjId = loadData(sparql, statements);
+
+ // Create the expected results of the SPARQL query once the PCJ has been computed.
+ final Set<VisibilityBindingSet> expectedResults = new HashSet<>();
+
+ MapBindingSet bs = new MapBindingSet();
+ bs.addBinding("type", vf.createLiteral("cheese", XMLSchema.STRING));
+ bs.addBinding("location", vf.createLiteral("France", XMLSchema.STRING));
+ bs.addBinding("averagePrice", vf.createLiteral("8.5", XMLSchema.DECIMAL));
+ expectedResults.add( new VisibilityBindingSet(bs));
+
+ bs = new MapBindingSet();
+ bs.addBinding("type", vf.createLiteral("cigarettes", XMLSchema.STRING));
+ bs.addBinding("location", vf.createLiteral("France", XMLSchema.STRING));
+ bs.addBinding("averagePrice", vf.createLiteral("4.49", XMLSchema.DECIMAL));
+ expectedResults.add( new VisibilityBindingSet(bs) );
+
+ bs = new MapBindingSet();
+ bs.addBinding("type", vf.createLiteral("cheese", XMLSchema.STRING));
+ bs.addBinding("location", vf.createLiteral("USA", XMLSchema.STRING));
+ bs.addBinding("averagePrice", vf.createLiteral("4.75", XMLSchema.DECIMAL));
+ expectedResults.add( new VisibilityBindingSet(bs) );
+
+ // Verify the end results of the query match the expected results.
+ final Set<VisibilityBindingSet> results = readGroupedResults(pcjId, new VariableOrder("type", "location"));
+ System.out.println(results);
+ assertEquals(expectedResults, results);
+ }
+
+
+ @Test
+ public void nestedWithJoinGroupByManyBindings_averages() throws Exception {
+
+ // A query that groups what is aggregated by two of the keys.
+ final String sparql =
+ "SELECT ?type ?location ?averagePrice ?milkType {" +
+ "FILTER(?averagePrice > 4) " +
+ "?type <urn:hasMilkType> ?milkType ." +
+ "{SELECT ?type ?location (avg(?price) as ?averagePrice) {" +
+ "?id <urn:type> ?type . " +
+ "?id <urn:location> ?location ." +
+ "?id <urn:price> ?price ." +
+ "} " +
+ "GROUP BY ?type ?location }}";
+
+ // Create the Statements that will be loaded into Rya.
+ final ValueFactory vf = new ValueFactoryImpl();
+ final Collection<Statement> statements = Sets.newHashSet(
+
+ vf.createStatement(vf.createURI("urn:1"), vf.createURI("urn:type"), vf.createURI("urn:blue")),
+ vf.createStatement(vf.createURI("urn:1"), vf.createURI("urn:location"), vf.createLiteral("France")),
+ vf.createStatement(vf.createURI("urn:1"), vf.createURI("urn:price"), vf.createLiteral(8.5)),
+ vf.createStatement(vf.createURI("urn:blue"), vf.createURI("urn:hasMilkType"), vf.createLiteral("cow", XMLSchema.STRING)),
+
+ vf.createStatement(vf.createURI("urn:2"), vf.createURI("urn:type"), vf.createURI("urn:american")),
+ vf.createStatement(vf.createURI("urn:2"), vf.createURI("urn:location"), vf.createLiteral("USA")),
+ vf.createStatement(vf.createURI("urn:2"), vf.createURI("urn:price"), vf.createLiteral(.99)),
+
+ vf.createStatement(vf.createURI("urn:3"), vf.createURI("urn:type"), vf.createURI("urn:cheddar")),
+ vf.createStatement(vf.createURI("urn:3"), vf.createURI("urn:location"), vf.createLiteral("USA")),
+ vf.createStatement(vf.createURI("urn:3"), vf.createURI("urn:price"), vf.createLiteral(5.25)),
+
+ // French items that will be averaged.
+ vf.createStatement(vf.createURI("urn:4"), vf.createURI("urn:type"), vf.createURI("urn:goat")),
+ vf.createStatement(vf.createURI("urn:4"), vf.createURI("urn:location"), vf.createLiteral("France")),
+ vf.createStatement(vf.createURI("urn:4"), vf.createURI("urn:price"), vf.createLiteral(6.5)),
+ vf.createStatement(vf.createURI("urn:goat"), vf.createURI("urn:hasMilkType"), vf.createLiteral("goat", XMLSchema.STRING)),
+
+ vf.createStatement(vf.createURI("urn:5"), vf.createURI("urn:type"), vf.createURI("urn:fontina")),
+ vf.createStatement(vf.createURI("urn:5"), vf.createURI("urn:location"), vf.createLiteral("Italy")),
+ vf.createStatement(vf.createURI("urn:5"), vf.createURI("urn:price"), vf.createLiteral(3.99)),
+ vf.createStatement(vf.createURI("urn:fontina"), vf.createURI("urn:hasMilkType"), vf.createLiteral("cow", XMLSchema.STRING)),
+
+ vf.createStatement(vf.createURI("urn:6"), vf.createURI("urn:type"), vf.createURI("urn:fontina")),
+ vf.createStatement(vf.createURI("urn:6"), vf.createURI("urn:location"), vf.createLiteral("Italy")),
+ vf.createStatement(vf.createURI("urn:6"), vf.createURI("urn:price"), vf.createLiteral(4.99)));
+
+ // Create the PCJ in Fluo and load the statements into Rya.
+ final String pcjId = loadData(sparql, statements);
+
+ // Create the expected results of the SPARQL query once the PCJ has been computed.
+ final Set<VisibilityBindingSet> expectedResults = new HashSet<>();
+
+ MapBindingSet bs = new MapBindingSet();
+ bs.addBinding("type", vf.createURI("urn:blue"));
+ bs.addBinding("location", vf.createLiteral("France", XMLSchema.STRING));
+ bs.addBinding("averagePrice", vf.createLiteral("8.5", XMLSchema.DECIMAL));
+ bs.addBinding("milkType", vf.createLiteral("cow", XMLSchema.STRING));
+ expectedResults.add( new VisibilityBindingSet(bs));
+
+ bs = new MapBindingSet();
+ bs.addBinding("type", vf.createURI("urn:goat"));
+ bs.addBinding("location", vf.createLiteral("France", XMLSchema.STRING));
+ bs.addBinding("averagePrice", vf.createLiteral("6.5", XMLSchema.DECIMAL));
+ bs.addBinding("milkType", vf.createLiteral("goat", XMLSchema.STRING));
+ expectedResults.add( new VisibilityBindingSet(bs) );
+
+ bs = new MapBindingSet();
+ bs.addBinding("type", vf.createURI("urn:fontina"));
+ bs.addBinding("location", vf.createLiteral("Italy", XMLSchema.STRING));
+ bs.addBinding("averagePrice", vf.createLiteral("4.49", XMLSchema.DECIMAL));
+ bs.addBinding("milkType", vf.createLiteral("cow", XMLSchema.STRING));
+ expectedResults.add( new VisibilityBindingSet(bs) );
+
+ // Verify the end results of the query match the expected results.
+ final Set<VisibilityBindingSet> results = readGroupedResults(pcjId, new VariableOrder("type", "location"));
+ System.out.println(results);
+ assertEquals(expectedResults, results);
+ }
+
private Set<VisibilityBindingSet> readAllResults(final String pcjId) throws Exception {
requireNonNull(pcjId);
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/KafkaRyaSubGraphExportIT.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/KafkaRyaSubGraphExportIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/KafkaRyaSubGraphExportIT.java
index 7a4ed8d..ca8de0d 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/KafkaRyaSubGraphExportIT.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/KafkaRyaSubGraphExportIT.java
@@ -43,24 +43,28 @@ import org.apache.kafka.common.serialization.StringSerializer;
import org.apache.rya.accumulo.AccumuloRyaDAO;
import org.apache.rya.api.domain.RyaStatement;
import org.apache.rya.api.domain.RyaSubGraph;
+import org.apache.rya.api.domain.RyaType;
import org.apache.rya.api.domain.RyaURI;
import org.apache.rya.api.resolver.RdfToRyaConversions;
import org.apache.rya.indexing.pcj.fluo.ConstructGraphTestUtils;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePcj;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
import org.apache.rya.indexing.pcj.fluo.app.export.kafka.KafkaExportParameters;
import org.apache.rya.indexing.pcj.fluo.app.export.kafka.RyaSubGraphKafkaSerDe;
import org.apache.rya.indexing.pcj.fluo.app.observers.AggregationObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.ConstructQueryResultObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.FilterObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.JoinObserver;
+import org.apache.rya.indexing.pcj.fluo.app.observers.ProjectionObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.StatementPatternObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.TripleObserver;
-import org.apache.rya.indexing.pcj.fluo.app.query.FluoQuery;
import org.apache.rya.pcj.fluo.test.base.KafkaExportITBase;
+import org.junit.Assert;
import org.junit.Test;
import org.openrdf.model.Statement;
import org.openrdf.model.ValueFactory;
import org.openrdf.model.impl.ValueFactoryImpl;
+import org.openrdf.model.vocabulary.RDF;
+import org.openrdf.model.vocabulary.XMLSchema;
import com.google.common.collect.Sets;
@@ -83,6 +87,7 @@ public class KafkaRyaSubGraphExportIT extends KafkaExportITBase {
observers.add(new ObserverSpecification(JoinObserver.class.getName()));
observers.add(new ObserverSpecification(FilterObserver.class.getName()));
observers.add(new ObserverSpecification(AggregationObserver.class.getName()));
+ observers.add(new ObserverSpecification(ProjectionObserver.class.getName()));
// Configure the export observer to export new PCJ results to the mini
// accumulo cluster.
@@ -285,6 +290,77 @@ public class KafkaRyaSubGraphExportIT extends KafkaExportITBase {
ConstructGraphTestUtils.subGraphsEqualIgnoresBlankNode(expectedResults, results);
}
+
+ @Test
+ public void nestedConstructQuery() throws Exception {
+ // A query that groups what is aggregated by one of the keys.
+ final String sparql = "CONSTRUCT { "
+ + "_:b a <urn:highSpeedTrafficArea> . "
+ + "_:b <urn:hasCount> ?obsCount . "
+ + "_:b <urn:hasLocation> ?location ."
+ + "_:b <urn:hasAverageVelocity> ?avgVelocity ."
+ + "} WHERE { "
+ + "FILTER(?obsCount > 1) "
+ + "{ "
+ + "SELECT ?location (count(?obs) AS ?obsCount) (avg(?velocity) AS ?avgVelocity) "
+ + "WHERE { "
+ + "FILTER(?velocity > 75) "
+ + "?obs <urn:hasVelocity> ?velocity. "
+ + "?obs <urn:hasLocation> ?location. "
+ + "}GROUP BY ?location }}";
+
+ // Create the Statements that will be loaded into Rya.
+ final ValueFactory vf = new ValueFactoryImpl();
+ final Collection<Statement> statements = Sets.newHashSet(
+ vf.createStatement(vf.createURI("urn:obs1"), vf.createURI("urn:hasVelocity"), vf.createLiteral(77)),
+ vf.createStatement(vf.createURI("urn:obs1"), vf.createURI("urn:hasLocation"), vf.createLiteral("OldTown")),
+ vf.createStatement(vf.createURI("urn:obs2"), vf.createURI("urn:hasVelocity"), vf.createLiteral(81)),
+ vf.createStatement(vf.createURI("urn:obs2"), vf.createURI("urn:hasLocation"), vf.createLiteral("OldTown")),
+ vf.createStatement(vf.createURI("urn:obs3"), vf.createURI("urn:hasVelocity"), vf.createLiteral(70)),
+ vf.createStatement(vf.createURI("urn:obs3"), vf.createURI("urn:hasLocation"), vf.createLiteral("OldTown")),
+ vf.createStatement(vf.createURI("urn:obs5"), vf.createURI("urn:hasVelocity"), vf.createLiteral(87)),
+ vf.createStatement(vf.createURI("urn:obs5"), vf.createURI("urn:hasLocation"), vf.createLiteral("Rosslyn")),
+ vf.createStatement(vf.createURI("urn:obs6"), vf.createURI("urn:hasVelocity"), vf.createLiteral(81)),
+ vf.createStatement(vf.createURI("urn:obs6"), vf.createURI("urn:hasLocation"), vf.createLiteral("Rosslyn")),
+ vf.createStatement(vf.createURI("urn:obs7"), vf.createURI("urn:hasVelocity"), vf.createLiteral(67)),
+ vf.createStatement(vf.createURI("urn:obs7"), vf.createURI("urn:hasLocation"), vf.createLiteral("Clarendon")),
+ vf.createStatement(vf.createURI("urn:obs8"), vf.createURI("urn:hasVelocity"), vf.createLiteral(77)),
+ vf.createStatement(vf.createURI("urn:obs8"), vf.createURI("urn:hasLocation"), vf.createLiteral("Ballston")),
+ vf.createStatement(vf.createURI("urn:obs9"), vf.createURI("urn:hasVelocity"), vf.createLiteral(87)),
+ vf.createStatement(vf.createURI("urn:obs9"), vf.createURI("urn:hasLocation"), vf.createLiteral("FallsChurch")));
+
+ // Create the PCJ in Fluo and load the statements into Rya.
+ final String pcjId = loadStatements(sparql, statements);
+
+ // Verify the end results of the query match the expected results.
+ final Set<RyaSubGraph> results = readAllResults(pcjId);
+
+ RyaStatement statement1 = new RyaStatement(new RyaURI("urn:obs1"), new RyaURI("urn:hasCount"), new RyaType(XMLSchema.INTEGER, "2"));
+ RyaStatement statement2 = new RyaStatement(new RyaURI("urn:obs1"), new RyaURI("urn:hasAverageVelocity"), new RyaType(XMLSchema.DECIMAL, "84"));
+ RyaStatement statement3 = new RyaStatement(new RyaURI("urn:obs1"), new RyaURI("urn:hasLocation"), new RyaType("Rosslyn"));
+ RyaStatement statement4 = new RyaStatement(new RyaURI("urn:obs1"), new RyaURI(RDF.TYPE.toString()), new RyaURI("urn:highSpeedTrafficArea"));
+ RyaStatement statement5 = new RyaStatement(new RyaURI("urn:obs2"), new RyaURI("urn:hasCount"), new RyaType(XMLSchema.INTEGER, "2"));
+ RyaStatement statement6 = new RyaStatement(new RyaURI("urn:obs2"), new RyaURI("urn:hasAverageVelocity"), new RyaType(XMLSchema.DECIMAL, "79"));
+ RyaStatement statement7 = new RyaStatement(new RyaURI("urn:obs2"), new RyaURI("urn:hasLocation"), new RyaType("OldTown"));
+ RyaStatement statement8 = new RyaStatement(new RyaURI("urn:obs2"), new RyaURI(RDF.TYPE.toString()), new RyaURI("urn:highSpeedTrafficArea"));
+
+ final Set<RyaSubGraph> expectedResults = new HashSet<>();
+
+ RyaSubGraph subGraph1 = new RyaSubGraph(pcjId);
+ Set<RyaStatement> stmnts1 = new HashSet<>(Arrays.asList(statement1, statement2, statement3, statement4));
+ subGraph1.setStatements(stmnts1);
+ expectedResults.add(subGraph1);
+
+ RyaSubGraph subGraph2 = new RyaSubGraph(pcjId);
+ Set<RyaStatement> stmnts2 = new HashSet<>(Arrays.asList(statement5, statement6, statement7, statement8));
+ subGraph2.setStatements(stmnts2);
+ expectedResults.add(subGraph2);
+
+ Assert.assertEquals(expectedResults.size(), results.size());
+ ConstructGraphTestUtils.subGraphsEqualIgnoresBlankNode(expectedResults, results);;
+ }
+
+
protected KafkaConsumer<String, RyaSubGraph> makeRyaSubGraphConsumer(final String TopicName) {
// setup consumer
final Properties consumerProps = new Properties();
@@ -330,9 +406,9 @@ public class KafkaRyaSubGraphExportIT extends KafkaExportITBase {
FluoClient client = null;
try {
- CreatePcj createPcj = new CreatePcj();
+ CreateFluoPcj createPcj = new CreateFluoPcj();
client = new FluoClientImpl(super.getFluoConfiguration());
- FluoQuery fluoQuery = createPcj.createFluoPcj(client, sparql);
+ String id = createPcj.createPcj(sparql, client).getQueryId();
AccumuloRyaDAO dao = getRyaDAO();
dao.add(statements.iterator());
@@ -341,7 +417,7 @@ public class KafkaRyaSubGraphExportIT extends KafkaExportITBase {
super.getMiniFluo().waitForObservers();
// FluoITHelper.printFluoTable(client);
- return fluoQuery.getConstructQueryMetadata().get().getNodeId();
+ return id;
} finally {
if (client != null) {
client.close();
[4/5] incubator-rya git commit: RYA-282-Nested-Query. Closes #192.
Posted by ca...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/IncrementalUpdateConstants.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/IncrementalUpdateConstants.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/IncrementalUpdateConstants.java
index 2084907..4b6f44e 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/IncrementalUpdateConstants.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/IncrementalUpdateConstants.java
@@ -35,9 +35,13 @@ public class IncrementalUpdateConstants {
public static final String FILTER_PREFIX = "FILTER";
public static final String AGGREGATION_PREFIX = "AGGREGATION";
public static final String QUERY_PREFIX = "QUERY";
+ public static final String PROJECTION_PREFIX = "PROJECTION";
public static final String CONSTRUCT_PREFIX = "CONSTRUCT";
public static final String PERIODIC_QUERY_PREFIX = "PERIODIC_QUERY";
+ public static enum QueryType{Construct, Projection, Periodic};
+ public static enum ExportStrategy{Rya, Kafka};
+
public static final String PERIODIC_BIN_ID = PeriodicQueryResultStorage.PeriodicBinId;
public static final String URI_TYPE = "http://www.w3.org/2001/XMLSchema#anyURI";
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/JoinResultUpdater.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/JoinResultUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/JoinResultUpdater.java
index 9b65b34..0f448a6 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/JoinResultUpdater.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/JoinResultUpdater.java
@@ -139,9 +139,9 @@ public class JoinResultUpdater {
// Create the Row Key for the emitted binding set. It does not contain visibilities.
final Bytes resultRow = RowKeyUtil.makeRowKey(joinMetadata.getNodeId(), joinVarOrder, newJoinResult);
-
- // Only insert the join Binding Set if it is new.
- if(tx.get(resultRow, FluoQueryColumns.JOIN_BINDING_SET) == null) {
+
+ // Only insert the join Binding Set if it is new or BindingSet contains values not used in resultRow.
+ if(tx.get(resultRow, FluoQueryColumns.JOIN_BINDING_SET) == null || joinVarOrder.getVariableOrders().size() < newJoinResult.size()) {
// Create the Node Value. It does contain visibilities.
final Bytes nodeValueBytes = BS_SERDE.serialize(newJoinResult);
@@ -210,18 +210,28 @@ public class JoinResultUpdater {
final NodeType nodeType = NodeType.fromNodeId(nodeId).get();
switch(nodeType) {
case STATEMENT_PATTERN:
- return queryDao.readStatementPatternMetadata(tx, nodeId).getVariableOrder();
-
+ return removeBinIdFromVarOrder(queryDao.readStatementPatternMetadata(tx, nodeId).getVariableOrder());
case FILTER:
- return queryDao.readFilterMetadata(tx, nodeId).getVariableOrder();
-
+ return removeBinIdFromVarOrder(queryDao.readFilterMetadata(tx, nodeId).getVariableOrder());
case JOIN:
- return queryDao.readJoinMetadata(tx, nodeId).getVariableOrder();
-
+ return removeBinIdFromVarOrder(queryDao.readJoinMetadata(tx, nodeId).getVariableOrder());
+ case PROJECTION:
+ return removeBinIdFromVarOrder(queryDao.readProjectionMetadata(tx, nodeId).getVariableOrder());
default:
throw new IllegalArgumentException("Could not figure out the variable order for node with ID: " + nodeId);
}
}
+
+ private VariableOrder removeBinIdFromVarOrder(VariableOrder varOrder) {
+ List<String> varOrderList = varOrder.getVariableOrders();
+ if(varOrderList.get(0).equals(IncrementalUpdateConstants.PERIODIC_BIN_ID)) {
+ List<String> updatedVarOrderList = Lists.newArrayList(varOrderList);
+ updatedVarOrderList.remove(0);
+ return new VariableOrder(updatedVarOrderList);
+ } else {
+ return varOrder;
+ }
+ }
/**
* Assuming that the common variables between two children are already
@@ -285,6 +295,9 @@ public class JoinResultUpdater {
case JOIN:
column = FluoQueryColumns.JOIN_BINDING_SET;
break;
+ case PROJECTION:
+ column = FluoQueryColumns.PROJECTION_BINDING_SET;
+ break;
default:
throw new IllegalArgumentException("The child node's sibling is not of type StatementPattern, Join, Left Join, or Filter.");
}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/NodeType.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/NodeType.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/NodeType.java
index b8fc2d9..a6fc5ea 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/NodeType.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/NodeType.java
@@ -24,10 +24,12 @@ import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.CO
import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.FILTER_PREFIX;
import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.JOIN_PREFIX;
import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.QUERY_PREFIX;
+import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.PROJECTION_PREFIX;
import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.SP_PREFIX;
import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.PERIODIC_QUERY_PREFIX;
import java.util.List;
+import java.util.UUID;
import org.apache.fluo.api.data.Column;
import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns;
@@ -39,13 +41,14 @@ import com.google.common.base.Optional;
* Represents the different types of nodes that a Query may have.
*/
public enum NodeType {
- PERIODIC_QUERY(QueryNodeMetadataColumns.PERIODIC_QUERY_COLUMNS, FluoQueryColumns.PERIODIC_QUERY_BINDING_SET),
- FILTER (QueryNodeMetadataColumns.FILTER_COLUMNS, FluoQueryColumns.FILTER_BINDING_SET),
- JOIN(QueryNodeMetadataColumns.JOIN_COLUMNS, FluoQueryColumns.JOIN_BINDING_SET),
- STATEMENT_PATTERN(QueryNodeMetadataColumns.STATEMENTPATTERN_COLUMNS, FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET),
- QUERY(QueryNodeMetadataColumns.QUERY_COLUMNS, FluoQueryColumns.QUERY_BINDING_SET),
- AGGREGATION(QueryNodeMetadataColumns.AGGREGATION_COLUMNS, FluoQueryColumns.AGGREGATION_BINDING_SET),
- CONSTRUCT(QueryNodeMetadataColumns.CONSTRUCT_COLUMNS, FluoQueryColumns.CONSTRUCT_STATEMENTS);
+ PERIODIC_QUERY(IncrementalUpdateConstants.PERIODIC_QUERY_PREFIX, QueryNodeMetadataColumns.PERIODIC_QUERY_COLUMNS, FluoQueryColumns.PERIODIC_QUERY_BINDING_SET),
+ FILTER (IncrementalUpdateConstants.FILTER_PREFIX, QueryNodeMetadataColumns.FILTER_COLUMNS, FluoQueryColumns.FILTER_BINDING_SET),
+ JOIN(IncrementalUpdateConstants.JOIN_PREFIX, QueryNodeMetadataColumns.JOIN_COLUMNS, FluoQueryColumns.JOIN_BINDING_SET),
+ STATEMENT_PATTERN(IncrementalUpdateConstants.SP_PREFIX, QueryNodeMetadataColumns.STATEMENTPATTERN_COLUMNS, FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET),
+ QUERY(IncrementalUpdateConstants.QUERY_PREFIX, QueryNodeMetadataColumns.QUERY_COLUMNS, FluoQueryColumns.QUERY_BINDING_SET),
+ AGGREGATION(IncrementalUpdateConstants.AGGREGATION_PREFIX, QueryNodeMetadataColumns.AGGREGATION_COLUMNS, FluoQueryColumns.AGGREGATION_BINDING_SET),
+ PROJECTION(IncrementalUpdateConstants.PROJECTION_PREFIX, QueryNodeMetadataColumns.PROJECTION_COLUMNS, FluoQueryColumns.PROJECTION_BINDING_SET),
+ CONSTRUCT(IncrementalUpdateConstants.CONSTRUCT_PREFIX, QueryNodeMetadataColumns.CONSTRUCT_COLUMNS, FluoQueryColumns.CONSTRUCT_STATEMENTS);
//Metadata Columns associated with given NodeType
private QueryNodeMetadataColumns metadataColumns;
@@ -53,15 +56,25 @@ public enum NodeType {
//Column where results are stored for given NodeType
private Column resultColumn;
+ //Prefix for the given node type
+ private String nodePrefix;
/**
* Constructs an instance of {@link NodeType}.
*
* @param metadataColumns - Metadata {@link Column}s associated with this {@link NodeType}. (not null)
* @param resultColumn - The {@link Column} used to store this {@link NodeType}'s results. (not null)
*/
- private NodeType(QueryNodeMetadataColumns metadataColumns, Column resultColumn) {
+ private NodeType(String nodePrefix, QueryNodeMetadataColumns metadataColumns, Column resultColumn) {
this.metadataColumns = requireNonNull(metadataColumns);
this.resultColumn = requireNonNull(resultColumn);
+ this.nodePrefix = requireNonNull(nodePrefix);
+ }
+
+ /**
+ * @return the prefix for the given node type
+ */
+ public String getNodeTypePrefix() {
+ return nodePrefix;
}
/**
@@ -103,10 +116,38 @@ public enum NodeType {
type = AGGREGATION;
} else if(nodeId.startsWith(CONSTRUCT_PREFIX)) {
type = CONSTRUCT;
+ } else if(nodeId.startsWith(PROJECTION_PREFIX)) {
+ type = PROJECTION;
} else if(nodeId.startsWith(PERIODIC_QUERY_PREFIX)) {
type = PERIODIC_QUERY;
}
return Optional.fromNullable(type);
}
+
+ /**
+ * Creates an id for a given NodeType that is of the form {@link NodeType#getNodeTypePrefix()} + "_" + pcjId,
+ * where the pcjId is an auto generated UUID with all dashes removed.
+ * @param type {@link NodeType}
+ * @return id for the given NodeType
+ */
+ public static String generateNewFluoIdForType(NodeType type) {
+ String unique = UUID.randomUUID().toString().replaceAll("-", "");
+ // Put them together to create the Node ID.
+ return type.getNodeTypePrefix() + "_" + unique;
+ }
+
+ /**
+ * Creates an id for a given NodeType that is of the form {@link NodeType#getNodeTypePrefix()} + "_" + pcjId
+ *
+ * @param type {@link NodeType}
+ * @return id for the given NodeType
+ */
+ public static String generateNewIdForType(NodeType type, String pcjId) {
+ // Put them together to create the Node ID.
+ return type.getNodeTypePrefix() + "_" + pcjId;
+ }
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/PeriodicQueryUpdater.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/PeriodicQueryUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/PeriodicQueryUpdater.java
index ae4912b..cb331cf 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/PeriodicQueryUpdater.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/PeriodicQueryUpdater.java
@@ -34,7 +34,6 @@ import org.openrdf.model.Literal;
import org.openrdf.model.Value;
import org.openrdf.model.ValueFactory;
import org.openrdf.model.impl.ValueFactoryImpl;
-import org.openrdf.query.Binding;
import org.openrdf.query.algebra.evaluation.QueryBindingSet;
/**
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ProjectionResultUpdater.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ProjectionResultUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ProjectionResultUpdater.java
new file mode 100644
index 0000000..f9d8257
--- /dev/null
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ProjectionResultUpdater.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.fluo.app;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.log4j.Logger;
+import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns;
+import org.apache.rya.indexing.pcj.fluo.app.query.ProjectionMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.util.BindingSetUtil;
+import org.apache.rya.indexing.pcj.fluo.app.util.RowKeyUtil;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSetSerDe;
+import org.openrdf.query.BindingSet;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
+
+/**
+ * Updates the results of a Projection node when one of its children has added a
+ * new Binding Set to its results.
+ */
+@DefaultAnnotation(NonNull.class)
+public class ProjectionResultUpdater {
+ private static final Logger log = Logger.getLogger(QueryResultUpdater.class);
+
+ private static final VisibilityBindingSetSerDe BS_SERDE = new VisibilityBindingSetSerDe();
+
+ /**
+ * Updates the results of a Projection node when one of its children has added a
+ * new Binding Set to its results.
+ *
+ * @param tx - The transaction all Fluo queries will use. (not null)
+ * @param childBindingSet - A binding set that the query's child node has emmitted. (not null)
+ * @param projectionMetadata - The metadata of the Query whose results will be updated. (not null)
+ * @throws Exception A problem caused the update to fail.
+ */
+ public void updateProjectionResults(
+ final TransactionBase tx,
+ final VisibilityBindingSet childBindingSet,
+ final ProjectionMetadata projectionMetadata) throws Exception {
+ checkNotNull(tx);
+ checkNotNull(childBindingSet);
+ checkNotNull(projectionMetadata);
+
+ log.trace(
+ "Transaction ID: " + tx.getStartTimestamp() + "\n" +
+ "Node ID: " + projectionMetadata.getNodeId() + "\n" +
+ "Parent Node ID: " + projectionMetadata.getParentNodeId() + "\n" +
+ "Child Node ID: " + projectionMetadata.getChildNodeId() + "\n" +
+ "Child Binding Set:\n" + childBindingSet + "\n");
+
+ // Create the query's Binding Set from the child node's binding set.
+ final VariableOrder queryVarOrder = projectionMetadata.getVariableOrder();
+ final VariableOrder projectionVarOrder = projectionMetadata.getProjectedVars();
+ final BindingSet queryBindingSet = BindingSetUtil.keepBindings(projectionVarOrder, childBindingSet);
+
+ // Create the Row Key for the result. If the child node groups results, then the key must only contain the Group By variables.
+ Bytes resultRow = RowKeyUtil.makeRowKey(projectionMetadata.getNodeId(), queryVarOrder, queryBindingSet);
+
+ // Create the Binding Set that goes in the Node Value. It does contain visibilities.
+ final Bytes nodeValueBytes = BS_SERDE.serialize(new VisibilityBindingSet(queryBindingSet, childBindingSet.getVisibility()));
+
+ log.trace(
+ "Transaction ID: " + tx.getStartTimestamp() + "\n" +
+ "New Binding Set: " + childBindingSet + "\n");
+
+ tx.set(resultRow, FluoQueryColumns.PROJECTION_BINDING_SET, nodeValueBytes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/QueryResultUpdater.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/QueryResultUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/QueryResultUpdater.java
index 44fc9bd..37d7256 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/QueryResultUpdater.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/QueryResultUpdater.java
@@ -23,16 +23,12 @@ import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.fluo.api.client.TransactionBase;
import org.apache.fluo.api.data.Bytes;
import org.apache.log4j.Logger;
-import org.apache.rya.indexing.pcj.fluo.app.query.AggregationMetadata;
import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns;
-import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryMetadataDAO;
import org.apache.rya.indexing.pcj.fluo.app.query.QueryMetadata;
-import org.apache.rya.indexing.pcj.fluo.app.util.BindingSetUtil;
import org.apache.rya.indexing.pcj.fluo.app.util.RowKeyUtil;
import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSetSerDe;
-import org.openrdf.query.BindingSet;
import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
import edu.umd.cs.findbugs.annotations.NonNull;
@@ -45,7 +41,6 @@ import edu.umd.cs.findbugs.annotations.NonNull;
public class QueryResultUpdater {
private static final Logger log = Logger.getLogger(QueryResultUpdater.class);
- private static final FluoQueryMetadataDAO METADATA_DA0 = new FluoQueryMetadataDAO();
private static final VisibilityBindingSetSerDe BS_SERDE = new VisibilityBindingSetSerDe();
/**
@@ -73,23 +68,12 @@ public class QueryResultUpdater {
// Create the query's Binding Set from the child node's binding set.
final VariableOrder queryVarOrder = queryMetadata.getVariableOrder();
- final BindingSet queryBindingSet = BindingSetUtil.keepBindings(queryVarOrder, childBindingSet);
// Create the Row Key for the result. If the child node groups results, then the key must only contain the Group By variables.
- final Bytes resultRow;
-
- final String childNodeId = queryMetadata.getChildNodeId();
- final boolean isGrouped = childNodeId.startsWith( IncrementalUpdateConstants.AGGREGATION_PREFIX );
- if(isGrouped) {
- final AggregationMetadata aggMetadata = METADATA_DA0.readAggregationMetadata(tx, childNodeId);
- final VariableOrder groupByVars = aggMetadata.getGroupByVariableOrder();
- resultRow = RowKeyUtil.makeRowKey(queryMetadata.getNodeId(), groupByVars, queryBindingSet);
- } else {
- resultRow = RowKeyUtil.makeRowKey(queryMetadata.getNodeId(), queryVarOrder, queryBindingSet);
- }
+ final Bytes resultRow = RowKeyUtil.makeRowKey(queryMetadata.getNodeId(), queryVarOrder, childBindingSet);
// Create the Binding Set that goes in the Node Value. It does contain visibilities.
- final Bytes nodeValueBytes = BS_SERDE.serialize(new VisibilityBindingSet(queryBindingSet,childBindingSet.getVisibility()));
+ final Bytes nodeValueBytes = BS_SERDE.serialize(childBindingSet);
log.trace(
"Transaction ID: " + tx.getStartTimestamp() + "\n" +
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/kafka/KafkaRyaSubGraphExporter.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/kafka/KafkaRyaSubGraphExporter.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/kafka/KafkaRyaSubGraphExporter.java
index a15743f..fa27b46 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/kafka/KafkaRyaSubGraphExporter.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/kafka/KafkaRyaSubGraphExporter.java
@@ -30,8 +30,6 @@ import org.apache.rya.api.domain.RyaSubGraph;
import org.apache.rya.indexing.pcj.fluo.app.export.IncrementalBindingSetExporter.ResultExportException;
import org.apache.rya.indexing.pcj.fluo.app.export.IncrementalRyaSubGraphExporter;
-import com.google.common.base.Preconditions;
-
/**
* Exports {@link RyaSubGraph}s to Kafka from Rya Fluo Application
*
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/BindingSetUpdater.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/BindingSetUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/BindingSetUpdater.java
index 3a731c2..7d0fd5e 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/BindingSetUpdater.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/BindingSetUpdater.java
@@ -31,6 +31,7 @@ import org.apache.rya.indexing.pcj.fluo.app.FilterResultUpdater;
import org.apache.rya.indexing.pcj.fluo.app.JoinResultUpdater;
import org.apache.rya.indexing.pcj.fluo.app.NodeType;
import org.apache.rya.indexing.pcj.fluo.app.PeriodicQueryUpdater;
+import org.apache.rya.indexing.pcj.fluo.app.ProjectionResultUpdater;
import org.apache.rya.indexing.pcj.fluo.app.QueryResultUpdater;
import org.apache.rya.indexing.pcj.fluo.app.query.AggregationMetadata;
import org.apache.rya.indexing.pcj.fluo.app.query.ConstructQueryMetadata;
@@ -38,6 +39,7 @@ import org.apache.rya.indexing.pcj.fluo.app.query.FilterMetadata;
import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryMetadataDAO;
import org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata;
import org.apache.rya.indexing.pcj.fluo.app.query.PeriodicQueryMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.ProjectionMetadata;
import org.apache.rya.indexing.pcj.fluo.app.query.QueryMetadata;
import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
@@ -61,6 +63,7 @@ public abstract class BindingSetUpdater extends AbstractObserver {
private final QueryResultUpdater queryUpdater = new QueryResultUpdater();
private final AggregationResultUpdater aggregationUpdater = new AggregationResultUpdater();
private final ConstructQueryResultUpdater constructUpdater = new ConstructQueryResultUpdater();
+ private final ProjectionResultUpdater projectionUpdater = new ProjectionResultUpdater();
private final PeriodicQueryUpdater periodicQueryUpdater = new PeriodicQueryUpdater();
@Override
@@ -107,6 +110,15 @@ public abstract class BindingSetUpdater extends AbstractObserver {
}
break;
+ case PROJECTION:
+ final ProjectionMetadata projectionQuery = queryDao.readProjectionMetadata(tx, parentNodeId);
+ try {
+ projectionUpdater.updateProjectionResults(tx, observedBindingSet, projectionQuery);
+ } catch (final Exception e) {
+ throw new RuntimeException("Could not process a Query node.", e);
+ }
+ break;
+
case CONSTRUCT:
final ConstructQueryMetadata constructQuery = queryDao.readConstructQueryMetadata(tx, parentNodeId);
try{
@@ -154,7 +166,7 @@ public abstract class BindingSetUpdater extends AbstractObserver {
default:
- throw new IllegalArgumentException("The parent node's NodeType must be of type Filter, Join, PeriodicBin or Query, but was " + parentNodeType);
+ throw new IllegalArgumentException("The parent node's NodeType must be of type Aggregation, Projection, ConstructQuery, Filter, Join, PeriodicBin or Query, but was " + parentNodeType);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/ProjectionObserver.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/ProjectionObserver.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/ProjectionObserver.java
new file mode 100644
index 0000000..b712606
--- /dev/null
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/ProjectionObserver.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.fluo.app.observers;
+
+import static java.util.Objects.requireNonNull;
+
+import org.apache.fluo.api.client.TransactionBase;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.log4j.Logger;
+import org.apache.rya.indexing.pcj.fluo.app.BindingSetRow;
+import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns;
+import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryMetadataDAO;
+import org.apache.rya.indexing.pcj.fluo.app.query.ProjectionMetadata;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
+import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSetSerDe;
+
+/**
+ * Performs incremental result exporting to the configured destinations.
+ */
+public class ProjectionObserver extends BindingSetUpdater {
+ private static final Logger log = Logger.getLogger(ProjectionObserver.class);
+
+ private static final VisibilityBindingSetSerDe BS_SERDE = new VisibilityBindingSetSerDe();
+ private final FluoQueryMetadataDAO queryDao = new FluoQueryMetadataDAO();
+
+ @Override
+ public ObservedColumn getObservedColumn() {
+ return new ObservedColumn(FluoQueryColumns.PROJECTION_BINDING_SET, NotificationType.STRONG);
+ }
+
+ @Override
+ public Observation parseObservation(final TransactionBase tx, final Bytes row) throws Exception {
+ requireNonNull(tx);
+ requireNonNull(row);
+
+ // Read the Filter metadata.
+ final String projectionNodeId = BindingSetRow.make(row).getNodeId();
+ final ProjectionMetadata projectionMetadata = queryDao.readProjectionMetadata(tx, projectionNodeId);
+
+ // Read the Visibility Binding Set from the value.
+ final Bytes valueBytes = tx.get(row, FluoQueryColumns.PROJECTION_BINDING_SET);
+ final VisibilityBindingSet projectionBindingSet = BS_SERDE.deserialize(valueBytes);
+
+ // Figure out which node needs to handle the new metadata.
+ final String parentNodeId = projectionMetadata.getParentNodeId();
+
+ return new Observation(projectionNodeId, projectionBindingSet, parentNodeId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/QueryResultObserver.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/QueryResultObserver.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/QueryResultObserver.java
index fbdca08..e6368ba 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/QueryResultObserver.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/QueryResultObserver.java
@@ -107,7 +107,7 @@ public class QueryResultObserver extends AbstractObserver {
// Read the Child Binding Set that will be exported.
final Bytes valueBytes = tx.get(brow, col);
final VisibilityBindingSet result = BS_SERDE.deserialize(valueBytes);
-
+
// Simplify the result's visibilities.
final String visibility = result.getVisibility();
if(!simplifiedVisibilities.containsKey(visibility)) {
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/AggregationMetadata.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/AggregationMetadata.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/AggregationMetadata.java
index ff42a0f..eaa072f 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/AggregationMetadata.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/AggregationMetadata.java
@@ -287,7 +287,7 @@ public class AggregationMetadata extends CommonNodeMetadata {
* Builds instances of {@link AggregationMetadata}.
*/
@DefaultAnnotation(NonNull.class)
- public static final class Builder {
+ public static final class Builder implements CommonNodeMetadata.Builder {
private final String nodeId;
private VariableOrder varOrder;
@@ -317,7 +317,7 @@ public class AggregationMetadata extends CommonNodeMetadata {
* single variable because aggregations are only able to emit the aggregated value.
* @return This builder so that method invocations may be chained.
*/
- public Builder setVariableOrder(@Nullable final VariableOrder varOrder) {
+ public Builder setVarOrder(@Nullable final VariableOrder varOrder) {
this.varOrder = varOrder;
return this;
}
@@ -350,6 +350,10 @@ public class AggregationMetadata extends CommonNodeMetadata {
this.childNodeId = childNodeId;
return this;
}
+
+ public String getChildNodeId() {
+ return childNodeId;
+ }
/**
* @param aggregation - An aggregation that will be performed over the BindingSets that are emitted from the child node.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/CommonNodeMetadata.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/CommonNodeMetadata.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/CommonNodeMetadata.java
index e54acf1..a20fe4d 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/CommonNodeMetadata.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/CommonNodeMetadata.java
@@ -99,4 +99,18 @@ public abstract class CommonNodeMetadata {
.append("}")
.toString();
}
+
+ /**
+ * Base interface for all metadata Builders. Using this type def
+ * allows for the implementation of a Builder visitor for navigating
+ * the Builder tree.
+ *
+ */
+ public static interface Builder {
+
+ public String getNodeId();
+
+ public VariableOrder getVariableOrder();
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/ConstructQueryMetadata.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/ConstructQueryMetadata.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/ConstructQueryMetadata.java
index e836c5d..6bf968e 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/ConstructQueryMetadata.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/ConstructQueryMetadata.java
@@ -38,7 +38,7 @@ public class ConstructQueryMetadata extends CommonNodeMetadata {
private String childNodeId;
private ConstructGraph graph;
- private String sparql;
+ private String parentNodeId;
/**
* Creates ConstructQueryMetadata object from the provided metadata arguments.
@@ -47,21 +47,11 @@ public class ConstructQueryMetadata extends CommonNodeMetadata {
* @param graph - {@link ConstructGraph} used to project {@link BindingSet}s onto sets of statement representing construct graph
* @param sparql - SPARQL query containing construct graph
*/
- public ConstructQueryMetadata(String nodeId, String childNodeId, ConstructGraph graph, String sparql) {
- super(nodeId, new VariableOrder("subject", "predicate", "object"));
- Preconditions.checkNotNull(childNodeId);
- Preconditions.checkNotNull(graph);
- Preconditions.checkNotNull(sparql);
- this.childNodeId = childNodeId;
- this.graph = graph;
- this.sparql = sparql;
- }
-
- /**
- * @return sparql query string representing this construct query
- */
- public String getSparql() {
- return sparql;
+ public ConstructQueryMetadata(String nodeId, String parentNodeId, String childNodeId, VariableOrder varOrder, ConstructGraph graph) {
+ super(nodeId, varOrder);
+ this.childNodeId = Preconditions.checkNotNull(childNodeId);
+ this.parentNodeId = Preconditions.checkNotNull(parentNodeId);
+ this.graph = Preconditions.checkNotNull(graph);
}
/**
@@ -71,6 +61,13 @@ public class ConstructQueryMetadata extends CommonNodeMetadata {
public String getChildNodeId() {
return childNodeId;
}
+
+ /**
+ * @return The parent of this construct node
+ */
+ public String getParentNodeId() {
+ return parentNodeId;
+ }
/**
* @return The ConstructGraph used to form statement {@link BindingSet}s for
@@ -82,7 +79,7 @@ public class ConstructQueryMetadata extends CommonNodeMetadata {
@Override
public int hashCode() {
- return Objects.hashCode(super.getNodeId(), super.getVariableOrder(), childNodeId, graph, sparql);
+ return Objects.hashCode(super.getNodeId(), super.getVariableOrder(), parentNodeId, childNodeId, graph);
}
@Override
@@ -94,8 +91,8 @@ public class ConstructQueryMetadata extends CommonNodeMetadata {
if (o instanceof ConstructQueryMetadata) {
ConstructQueryMetadata queryMetadata = (ConstructQueryMetadata) o;
if (super.equals(queryMetadata)) {
- return new EqualsBuilder().append(childNodeId, queryMetadata.childNodeId).append(graph, queryMetadata.graph)
- .append(sparql, queryMetadata.sparql).isEquals();
+ return new EqualsBuilder().append(parentNodeId, queryMetadata.parentNodeId).append(childNodeId, queryMetadata.childNodeId).append(graph, queryMetadata.graph)
+ .isEquals();
}
return false;
}
@@ -105,7 +102,7 @@ public class ConstructQueryMetadata extends CommonNodeMetadata {
@Override
public String toString() {
return new StringBuilder().append("Construct Query Metadata {\n").append(" Node ID: " + super.getNodeId() + "\n")
- .append(" SPARQL QUERY: " + sparql + "\n").append(" Variable Order: " + super.getVariableOrder() + "\n")
+ .append(" Variable Order: " + super.getVariableOrder() + "\n")
.append(" Child Node ID: " + childNodeId + "\n").append(" Construct Graph: " + graph.getProjections() + "\n")
.append("}").toString();
}
@@ -123,13 +120,14 @@ public class ConstructQueryMetadata extends CommonNodeMetadata {
* Builds instances of {@link QueryMetadata}.
*/
@DefaultAnnotation(NonNull.class)
- public static final class Builder {
+ public static final class Builder implements CommonNodeMetadata.Builder {
private String nodeId;
private ConstructGraph graph;
+ private String parentNodeId;
private String childNodeId;
- private String sparql;
+ private VariableOrder varOrder;
/**
* Set the node Id that identifies this Construct Query Node
@@ -144,21 +142,31 @@ public class ConstructQueryMetadata extends CommonNodeMetadata {
}
/**
- * Set the SPARQL String representing this construct query
- * @param SPARQL string representing this construct query
+ * @return the node id for this construct query
+ */
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ /**
+ * Sets the VariableOrder that determines how results will be written
+ * @param varOrder
+ * @return This builder so that method invocations may be chained.
*/
- public Builder setSparql(String sparql) {
- this.sparql = sparql;
+ public Builder setVarOrder(VariableOrder varOrder) {
+ this.varOrder = varOrder;
return this;
}
+
+ @Override
+ public VariableOrder getVariableOrder() {
+ return varOrder;
+ }
/**
- * Set the ConstructGraph used to form statement {@link BindingSet}s for
- * this Construct Query
+ * Set the ConstructGraph used to form statement {@link BindingSet}s for this Construct Query
*
- * @param varOrder
- * - ConstructGraph to project {@link BindingSet}s onto RDF
- * statements
+ * @param varOrder - ConstructGraph to project {@link BindingSet}s onto RDF statements
* @return This builder so that method invocations may be chained.
*/
public Builder setConstructGraph(ConstructGraph graph) {
@@ -167,25 +175,37 @@ public class ConstructQueryMetadata extends CommonNodeMetadata {
}
/**
- * Set the node whose results are projected onto the given
- * {@link ConstructGraph}.
+ * Set the node whose results are projected onto the given {@link ConstructGraph}.
*
- * @param childNodeId
- * - The node whose results are projected onto the given
- * {@link ConstructGraph}.
+ * @param childNodeId - The node whose results are projected onto the given {@link ConstructGraph}.
* @return This builder so that method invocations may be chained.
*/
public Builder setChildNodeId(String childNodeId) {
this.childNodeId = childNodeId;
return this;
}
+
+ public String getChildNodeId() {
+ return childNodeId;
+ }
+
+ /**
+ * Set the parent node of this {@link ConstructGraph}.
+ *
+ * @param parentNodeId - The the parent node of this {@link ConstructGraph}.
+ * @return This builder so that method invocations may be chained.
+ */
+ public Builder setParentNodeId(String parentNodeId) {
+ this.parentNodeId = parentNodeId;
+ return this;
+ }
/**
* @return An instance of {@link ConstructQueryMetadata} build using
* this builder's values.
*/
public ConstructQueryMetadata build() {
- return new ConstructQueryMetadata(nodeId, childNodeId, graph, sparql);
+ return new ConstructQueryMetadata(nodeId, parentNodeId, childNodeId, varOrder, graph);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FilterMetadata.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FilterMetadata.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FilterMetadata.java
index 7e2e995..a821d8c 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FilterMetadata.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FilterMetadata.java
@@ -145,7 +145,7 @@ public class FilterMetadata extends CommonNodeMetadata {
* Builds instances of {@link FilterMetadata}.
*/
@DefaultAnnotation(NonNull.class)
- public static final class Builder {
+ public static final class Builder implements CommonNodeMetadata.Builder{
private final String nodeId;
private VariableOrder varOrder;
@@ -179,6 +179,11 @@ public class FilterMetadata extends CommonNodeMetadata {
this.varOrder = varOrder;
return this;
}
+
+ @Override
+ public VariableOrder getVariableOrder() {
+ return varOrder;
+ }
/**
* Set the original SPARQL query the filter is derived from.
@@ -212,6 +217,10 @@ public class FilterMetadata extends CommonNodeMetadata {
this.childNodeId = childNodeId;
return this;
}
+
+ public String getChildNodeId() {
+ return childNodeId;
+ }
/**
* @return Returns an instance of {@link FilterMetadata} using this builder's values.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQuery.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQuery.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQuery.java
index 8d218af..65db02c 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQuery.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQuery.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.QueryType;
import com.google.common.base.Objects;
import com.google.common.base.Optional;
@@ -44,7 +45,8 @@ import net.jcip.annotations.Immutable;
@DefaultAnnotation(NonNull.class)
public class FluoQuery {
- private final Optional<QueryMetadata> queryMetadata;
+ private final QueryMetadata queryMetadata;
+ private final ImmutableMap<String, ProjectionMetadata> projectionMetadata;
private final Optional<ConstructQueryMetadata> constructMetadata;
private final Optional<PeriodicQueryMetadata> periodicQueryMetadata;
private final ImmutableMap<String, StatementPatternMetadata> statementPatternMetadata;
@@ -52,13 +54,15 @@ public class FluoQuery {
private final ImmutableMap<String, JoinMetadata> joinMetadata;
private final ImmutableMap<String, AggregationMetadata> aggregationMetadata;
private final QueryType type;
- public static enum QueryType {Projection, Construct};
+ private final String queryId;
/**
* Constructs an instance of {@link FluoQuery}. Private because applications
* must use {@link Builder} instead.
*
- * @param queryMetadata - The root node of a query that is updated in Fluo. (not null)
+ * @param queryMetadata - metadata for the query for handling results (not null)
+ * @param projectionMetadata - projection nodes of query that project results (not null)
+ * @param constructMetadata - construct node of query that creates subgraphs
* @param periodicQueryMetadata - The periodic query node that is updated in Fluo.
* @param statementPatternMetadata - A map from Node ID to Statement Pattern metadata as
* it is represented within the Fluo app. (not null)
@@ -71,52 +75,27 @@ public class FluoQuery {
*/
private FluoQuery(
final QueryMetadata queryMetadata,
+ final ImmutableMap<String, ProjectionMetadata> projectionMetadata,
+ final Optional<ConstructQueryMetadata> constructMetadata,
final Optional<PeriodicQueryMetadata> periodicQueryMetadata,
final ImmutableMap<String, StatementPatternMetadata> statementPatternMetadata,
final ImmutableMap<String, FilterMetadata> filterMetadata,
final ImmutableMap<String, JoinMetadata> joinMetadata,
final ImmutableMap<String, AggregationMetadata> aggregationMetadata) {
this.aggregationMetadata = requireNonNull(aggregationMetadata);
- this.queryMetadata = Optional.of(requireNonNull(queryMetadata));
- this.constructMetadata = Optional.absent();
+ this.queryMetadata = requireNonNull(queryMetadata);
+ this.queryId = queryMetadata.getNodeId();
+ this.projectionMetadata = requireNonNull(projectionMetadata);
+ this.constructMetadata = constructMetadata;
this.periodicQueryMetadata = periodicQueryMetadata;
this.statementPatternMetadata = requireNonNull(statementPatternMetadata);
this.filterMetadata = requireNonNull(filterMetadata);
this.joinMetadata = requireNonNull(joinMetadata);
- this.type = QueryType.Projection;
- }
-
-
- /**
- * Constructs an instance of {@link FluoQuery}. Private because applications
- * must use {@link Builder} instead.
- *
- * @param constructMetadata - The root node of a query that is updated in Fluo. (not null)
- * @param periodicQueryMetadata - The periodic query node that is updated in Fluo.
- * @param statementPatternMetadata - A map from Node ID to Statement Pattern metadata as
- * it is represented within the Fluo app. (not null)
- * @param filterMetadata A map from Node ID to Filter metadata as it is represented
- * within the Fluo app. (not null)
- * @param joinMetadata - A map from Node ID to Join metadata as it is represented
- * within the Fluo app. (not null)
- * @param aggregationMetadata - A map from Node ID to Aggregation metadata as it is
- * represented within the Fluo app. (not null)
- */
- private FluoQuery(
- final ConstructQueryMetadata constructMetadata,
- final Optional<PeriodicQueryMetadata> periodicQueryMetadata,
- final ImmutableMap<String, StatementPatternMetadata> statementPatternMetadata,
- final ImmutableMap<String, FilterMetadata> filterMetadata,
- final ImmutableMap<String, JoinMetadata> joinMetadata,
- final ImmutableMap<String, AggregationMetadata> aggregationMetadata) {
- this.constructMetadata = Optional.of(requireNonNull(constructMetadata));
- this.queryMetadata = Optional.absent();
- this.periodicQueryMetadata = periodicQueryMetadata;
- this.statementPatternMetadata = requireNonNull(statementPatternMetadata);
- this.filterMetadata = requireNonNull(filterMetadata);
- this.joinMetadata = requireNonNull(joinMetadata);
- this.aggregationMetadata = aggregationMetadata;
- this.type = QueryType.Construct;
+ if(constructMetadata.isPresent()) {
+ this.type = QueryType.Construct;
+ } else {
+ this.type = QueryType.Projection;
+ }
}
/**
@@ -126,24 +105,86 @@ public class FluoQuery {
public QueryType getQueryType() {
return type;
}
+
+ /**
+ * @return the unique id of this query
+ */
+ public String getQueryId() {
+ return queryId;
+ }
/**
* @return Metadata about the root node of a query that is updated within the Fluo app.
*/
- public Optional<QueryMetadata> getQueryMetadata() {
+ public QueryMetadata getQueryMetadata() {
return queryMetadata;
}
+ /**
+ * @param nodeId - node id of the query metadata
+ * @return Optional containing the queryMetadata if it matches the specified nodeId
+ */
+ public Optional<QueryMetadata> getQueryMetadata(String nodeId) {
+ if(queryMetadata.getNodeId().equals(nodeId)) {
+ return Optional.of(queryMetadata);
+ } else {
+ return Optional.absent();
+ }
+ }
+
+ /**
+ * @return construct query metadata for generating subgraphs
+ */
public Optional<ConstructQueryMetadata> getConstructQueryMetadata() {
return constructMetadata;
}
/**
+ * @param nodeId - node id of the ConstructMetadata
+ * @return Optional containing the ConstructMetadata if it is present and has the given nodeId
+ */
+ public Optional<ConstructQueryMetadata> getConstructQueryMetadata(String nodeId) {
+ if(constructMetadata.isPresent() && constructMetadata.get().getNodeId().equals(nodeId)) {
+ return constructMetadata;
+ } else {
+ return Optional.absent();
+ }
+ }
+
+ /**
+ * @param nodeId - id of the Projection metadata you want (not null)
+ * @return projection metadata corresponding to give nodeId
+ */
+ public Optional<ProjectionMetadata> getProjectionMetadata(String nodeId) {
+ return Optional.fromNullable(projectionMetadata.get(nodeId));
+ }
+
+ /**
+ * @return All of the projection metadata that is stored for the query
+ */
+ public Collection<ProjectionMetadata> getProjectionMetadata() {
+ return projectionMetadata.values();
+ }
+
+ /**
* @return All of the Periodic Query metadata that is stored for the query.
*/
public Optional<PeriodicQueryMetadata> getPeriodicQueryMetadata() {
return periodicQueryMetadata;
}
+
+ /**
+ * @param nodeId - id of the PeriodicQueryMetadata
+ * @return Optional containing the PeriodicQueryMetadata if it is present and has the given nodeId
+ */
+ public Optional<PeriodicQueryMetadata> getPeriodicQueryMetadata(String nodeId) {
+
+ if(periodicQueryMetadata.isPresent() && periodicQueryMetadata.get().getNodeId().equals(nodeId)) {
+ return periodicQueryMetadata;
+ } else {
+ return Optional.absent();
+ }
+ }
/**
* Get a Statement Pattern node's metadata.
@@ -254,8 +295,11 @@ public class FluoQuery {
public String toString() {
final StringBuilder builder = new StringBuilder();
- if(queryMetadata.isPresent()) {
- builder.append( queryMetadata.get().toString() );
+ builder.append(queryMetadata.toString());
+ builder.append("\n");
+
+ for(final ProjectionMetadata metadata : projectionMetadata.values()) {
+ builder.append(metadata);
builder.append("\n");
}
@@ -305,9 +349,10 @@ public class FluoQuery {
@DefaultAnnotation(NonNull.class)
public static final class Builder {
- private QueryMetadata.Builder queryBuilder = null;
- private ConstructQueryMetadata.Builder constructBuilder = null;
- private PeriodicQueryMetadata.Builder periodicQueryBuilder = null;
+ private QueryMetadata.Builder queryBuilder;
+ private ConstructQueryMetadata.Builder constructBuilder;
+ private PeriodicQueryMetadata.Builder periodicQueryBuilder;
+ private final Map<String, ProjectionMetadata.Builder> projectionBuilders = new HashMap<>();
private final Map<String, StatementPatternMetadata.Builder> spBuilders = new HashMap<>();
private final Map<String, FilterMetadata.Builder> filterBuilders = new HashMap<>();
private final Map<String, JoinMetadata.Builder> joinBuilders = new HashMap<>();
@@ -319,23 +364,55 @@ public class FluoQuery {
* @param queryBuilder - The builder representing the query's results.
* @return This builder so that method invocation may be chained.
*/
- public Builder setQueryMetadata(@Nullable final QueryMetadata.Builder queryBuilder) {
- this.queryBuilder = queryBuilder;
+ public Builder setQueryMetadata(final QueryMetadata.Builder queryBuilder) {
+ this.queryBuilder = requireNonNull(queryBuilder);
return this;
}
/**
* @return The Query metadata builder if one has been set.
*/
- public Optional<QueryMetadata.Builder> getQueryBuilder() {
- return Optional.fromNullable( queryBuilder );
+ public QueryMetadata.Builder getQueryBuilder() {
+ return queryBuilder;
+ }
+
+ /**
+ * @param nodeId - id of the QueryMetadata.Builder
+ * @return Optional containing the QueryMetadata.Builder if it has the specified nodeId
+ */
+ public Optional<QueryMetadata.Builder> getQueryBuilder(String nodeId) {
+ if(queryBuilder.getNodeId().equals(nodeId)) {
+ return Optional.of(queryBuilder);
+ } else {
+ return Optional.absent();
+ }
+
+ }
+
+ /**
+ * Sets the {@link ProjectionMetadata.Builder} that is used by this builder.
+ *
+ * @param projectionBuilder - The builder representing this query's projection
+ * @return This builder so that method invocation may be chained.
+ */
+ public Builder addProjectionBuilder(@Nullable final ProjectionMetadata.Builder projectionBuilder) {
+ requireNonNull(projectionBuilder);
+ projectionBuilders.put(projectionBuilder.getNodeId(), projectionBuilder);
+ return this;
+ }
+
+ /**
+ * @return The ProjectionMetadata builder if one has been set.
+ */
+ public Optional<ProjectionMetadata.Builder> getProjectionBuilder(String nodeId) {
+ requireNonNull(nodeId);
+ return Optional.fromNullable( projectionBuilders.get(nodeId) );
}
/**
* Sets the {@link ConstructQueryMetadata.Builder} that is used by this builder.
*
- * @param constructBuilder
- * - The builder representing the query's results.
+ * @param constructBuilder - The builder representing the query's results.
* @return This builder so that method invocation may be chained.
*/
public Builder setConstructQueryMetadata(@Nullable final ConstructQueryMetadata.Builder constructBuilder) {
@@ -344,6 +421,18 @@ public class FluoQuery {
}
/**
+ * @param id of the ConstructQueryMetadata.Builder
+ * @return Optional containing the ConstructQueryMetadata.Builder if it has been set and has the given nodeId.
+ */
+ public Optional<ConstructQueryMetadata.Builder> getConstructQueryBuilder(String nodeId) {
+ if(constructBuilder != null && constructBuilder.getNodeId().equals(nodeId)) {
+ return Optional.of(constructBuilder);
+ } else {
+ return Optional.absent();
+ }
+ }
+
+ /**
* @return The Construct Query metadata builder if one has been set.
*/
public Optional<ConstructQueryMetadata.Builder> getConstructQueryBuilder() {
@@ -442,8 +531,6 @@ public class FluoQuery {
this.aggregationBuilders.put(aggregationBuilder.getNodeId(), aggregationBuilder);
return this;
}
-
-
/**
* Adds a new {@link PeriodicQueryMetadata.Builder} to this builder.
@@ -457,7 +544,6 @@ public class FluoQuery {
return this;
}
-
/**
* Get a PeriodicQuery builder from this builder.
*
@@ -467,24 +553,35 @@ public class FluoQuery {
return Optional.fromNullable( periodicQueryBuilder);
}
-
/**
- * @return Creates a {@link FluoQuery} using the values that have been supplied to this builder.
+ * @param - id of the PeriodicQueryMetadata.Builder
+ * @return - Optional containing the PeriodicQueryMetadata.Builder if one has been set and it has the given nodeId
*/
- public FluoQuery build() {
- checkArgument((queryBuilder != null && constructBuilder == null) || (queryBuilder == null && constructBuilder != null));
+ public Optional<PeriodicQueryMetadata.Builder> getPeriodicQueryBuilder(String nodeId) {
- Optional<QueryMetadata.Builder> optionalQueryBuilder = getQueryBuilder();
- QueryMetadata queryMetadata = null;
- if(optionalQueryBuilder.isPresent()) {
- queryMetadata = optionalQueryBuilder.get().build();
+ if(periodicQueryBuilder != null && periodicQueryBuilder.getNodeId().equals(nodeId)) {
+ return Optional.of(periodicQueryBuilder);
+ } else {
+ return Optional.absent();
}
+ }
+
+ /**
+ * @return Creates a {@link FluoQuery} using the values that have been supplied to this builder.
+ */
+ public FluoQuery build() {
+ checkArgument((projectionBuilders.size() > 0 || constructBuilder != null));
Optional<PeriodicQueryMetadata.Builder> optionalPeriodicQueryBuilder = getPeriodicQueryBuilder();
PeriodicQueryMetadata periodicQueryMetadata = null;
if(optionalPeriodicQueryBuilder.isPresent()) {
periodicQueryMetadata = optionalPeriodicQueryBuilder.get().build();
}
+
+ final ImmutableMap.Builder<String, ProjectionMetadata> projectionMetadata = ImmutableMap.builder();
+ for(final Entry<String, ProjectionMetadata.Builder> entry : projectionBuilders.entrySet()) {
+ projectionMetadata.put(entry.getKey(), entry.getValue().build());
+ }
final ImmutableMap.Builder<String, StatementPatternMetadata> spMetadata = ImmutableMap.builder();
for(final Entry<String, StatementPatternMetadata.Builder> entry : spBuilders.entrySet()) {
@@ -506,12 +603,13 @@ public class FluoQuery {
aggregateMetadata.put(entry.getKey(), entry.getValue().build());
}
- if(queryBuilder != null) {
- return new FluoQuery(queryBuilder.build(), Optional.fromNullable(periodicQueryMetadata), spMetadata.build(), filterMetadata.build(), joinMetadata.build(), aggregateMetadata.build());
- }
- //constructBuilder non-null in this case, but no need to check
- else {
- return new FluoQuery(constructBuilder.build(), Optional.fromNullable(periodicQueryMetadata), spMetadata.build(), filterMetadata.build(), joinMetadata.build(), aggregateMetadata.build());
+ if(constructBuilder != null) {
+ if(periodicQueryMetadata != null) {
+ throw new IllegalArgumentException("Queries containing sliding window filters and construct query patterns are not supported.");
+ }
+ return new FluoQuery(queryBuilder.build(), projectionMetadata.build(), Optional.of(constructBuilder.build()), Optional.fromNullable(periodicQueryMetadata), spMetadata.build(), filterMetadata.build(), joinMetadata.build(), aggregateMetadata.build());
+ } else {
+ return new FluoQuery(queryBuilder.build(), projectionMetadata.build(), Optional.absent(), Optional.fromNullable(periodicQueryMetadata), spMetadata.build(), filterMetadata.build(), joinMetadata.build(), aggregateMetadata.build());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryColumns.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryColumns.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryColumns.java
index ed18d49..8cd25d0 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryColumns.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryColumns.java
@@ -42,6 +42,20 @@ import edu.umd.cs.findbugs.annotations.NonNull;
* <tr> <td>Node ID</td> <td>queryMetadata:variableOrder</td> <td>The Variable Order binding sets are emitted with.</td> </tr>
* <tr> <td>Node ID</td> <td>queryMetadata:sparql</td> <td>The original SPARQL query that is being computed by this query.</td> </tr>
* <tr> <td>Node ID</td> <td>queryMetadata:childNodeId</td> <td>The Node ID of the child who feeds this node.</td> </tr>
+ * <tr> <td>Node ID</td> <td>queryMetadata:queryType</td> <td>The {@link QueryType} of this query.</td> </tr>
+ * <tr> <td>Node ID</td> <td>queryMetadata:exportStrategies</td> <td>Strategies for exporting results from Rya Fluo app</td> </tr>
+ * <tr> <td>Node ID + DELIM + Binding Set String</td> <td>queryMetadata:bindingSet</td> <td>A {@link VisibilityBindingSet} object.</td> </tr>
+ * </table>
+ * </p>
+ * <p>
+ * <b>Projection Metadata</b>
+ * <table border="1" style="width:100%">
+ * <tr> <th>Fluo Row</td> <th>Fluo Column</td> <th>Fluo Value</td> </tr>
+ * <tr> <td>Node ID</td> <td>projectionMetadata:nodeId</td> <td>The Node ID of the Query.</td> </tr>
+ * <tr> <td>Node ID</td> <td>projectionMetadata:projectedVars</td> <td>The variables that results are projected onto.</td> </tr>*
+ * <tr> <td>Node ID</td> <td>projectionMetadata:variableOrder</td> <td>The Variable Order that Binding values are written in in the Row to identify solutions.</td> </tr>
+ * <tr> <td>Node ID</td> <td>projectionMetadata:childNodeId</td> <td>The Node ID of the child who feeds this node.</td> </tr>
+ * <tr> <td>Node ID</td> <td>projectionMetadata:parentNodeId</td> <td>The Node ID of the parent of this node.</td> </tr>
* <tr> <td>Node ID + DELIM + Binding Set String</td> <td>queryMetadata:bindingSet</td> <td>A {@link VisibilityBindingSet} object.</td> </tr>
* </table>
* </p>
@@ -50,11 +64,11 @@ import edu.umd.cs.findbugs.annotations.NonNull;
* <table border="1" style="width:100%">
* <tr> <th>Fluo Row</td> <th>Fluo Column</td> <th>Fluo Value</td> </tr>
* <tr> <td>Node ID</td> <td>constructMetadata:nodeId</td> <td>The Node ID of the Query.</td> </tr>
- * <tr> <td>Node ID</td> <td>constructMetadata:sparql</td> <td>The original SPARQL query that is being computed by this query.</td> </tr>
* <tr> <td>Node ID</td> <td>constructMetadata:variableOrder</td> <td>The Variable Order binding sets are emitted with.</td> </tr>
* <tr> <td>Node ID</td> <td>constructMetadata:graph</td> <td>The construct graph used to project BindingSets to statements.</td> </tr>
* <tr> <td>Node ID</td> <td>constructMetadata:childNodeId</td> <td>The Node ID of the child who feeds this node.</td> </tr>
- * <tr> <td>Node ID</td> <td>constructMetadata:statements</td> <td>The RDF statements produced by this construct query node.</td> </tr>
+ * <tr> <td>Node ID</td> <td>constructMetadata:parentNodeId</td> <td>The Node ID of the parent that this node feeds.</td> </tr>
+ * <tr> <td>Node ID + DELIM + Binding Set String</td> <td>constructMetadata:statements</td> <td>The RDF statements produced by this construct query node.</td> </tr>
* </table>
* </p>
* <p>
@@ -131,6 +145,7 @@ public class FluoQueryColumns {
public static final String STATEMENT_PATTERN_METADATA_CF = "statementPatternMetadata";
public static final String AGGREGATION_METADATA_CF = "aggregationMetadata";
public static final String CONSTRUCT_METADATA_CF = "constructMetadata";
+ public static final String PROJECTION_METADATA_CF = "projectionMetadata";
public static final String PERIODIC_QUERY_METADATA_CF = "periodicQueryMetadata";
/**
@@ -178,14 +193,24 @@ public class FluoQueryColumns {
public static final Column QUERY_SPARQL = new Column(QUERY_METADATA_CF, "sparql");
public static final Column QUERY_CHILD_NODE_ID = new Column(QUERY_METADATA_CF, "childNodeId");
public static final Column QUERY_BINDING_SET = new Column(QUERY_METADATA_CF, "bindingSet");
+ public static final Column QUERY_EXPORT_STRATEGIES = new Column(QUERY_METADATA_CF, "exportStrategies");
+ public static final Column QUERY_TYPE = new Column(QUERY_METADATA_CF, "queryType");
+
+ // Query Metadata columns.
+ public static final Column PROJECTION_NODE_ID = new Column(PROJECTION_METADATA_CF, "nodeId");
+ public static final Column PROJECTION_PROJECTED_VARS = new Column(PROJECTION_METADATA_CF, "projectedVars");
+ public static final Column PROJECTION_VARIABLE_ORDER = new Column(PROJECTION_METADATA_CF, "variableOrder");
+ public static final Column PROJECTION_CHILD_NODE_ID = new Column(PROJECTION_METADATA_CF, "childNodeId");
+ public static final Column PROJECTION_PARENT_NODE_ID = new Column(PROJECTION_METADATA_CF, "parentNodeId");
+ public static final Column PROJECTION_BINDING_SET = new Column(PROJECTION_METADATA_CF, "bindingSet");
// Construct Query Metadata columns.
public static final Column CONSTRUCT_NODE_ID = new Column(CONSTRUCT_METADATA_CF, "nodeId");
public static final Column CONSTRUCT_VARIABLE_ORDER = new Column(CONSTRUCT_METADATA_CF, "variableOrder");
public static final Column CONSTRUCT_GRAPH = new Column(CONSTRUCT_METADATA_CF, "graph");
public static final Column CONSTRUCT_CHILD_NODE_ID = new Column(CONSTRUCT_METADATA_CF, "childNodeId");
+ public static final Column CONSTRUCT_PARENT_NODE_ID = new Column(CONSTRUCT_METADATA_CF, "parentNodeId");
public static final Column CONSTRUCT_STATEMENTS = new Column(CONSTRUCT_METADATA_CF, "statements");
- public static final Column CONSTRUCT_SPARQL = new Column(CONSTRUCT_METADATA_CF, "sparql");
// Filter Metadata columns.
public static final Column FILTER_NODE_ID = new Column(FILTER_METADATA_CF, "nodeId");
@@ -256,8 +281,20 @@ public class FluoQueryColumns {
Arrays.asList(QUERY_NODE_ID,
QUERY_VARIABLE_ORDER,
QUERY_SPARQL,
+ QUERY_TYPE,
+ QUERY_EXPORT_STRATEGIES,
QUERY_CHILD_NODE_ID)),
+ /**
+ * The columns a {@link ProjectionMetadata} object's fields are stored within.
+ */
+ PROJECTION_COLUMNS(
+ Arrays.asList(PROJECTION_NODE_ID,
+ PROJECTION_PROJECTED_VARS,
+ PROJECTION_VARIABLE_ORDER,
+ PROJECTION_PARENT_NODE_ID,
+ PROJECTION_CHILD_NODE_ID)),
+
/**
* The columns a {@link PeriodicBinMetadata} object's fields are stored within.
@@ -280,7 +317,7 @@ public class FluoQueryColumns {
CONSTRUCT_VARIABLE_ORDER,
CONSTRUCT_GRAPH,
CONSTRUCT_CHILD_NODE_ID,
- CONSTRUCT_SPARQL,
+ CONSTRUCT_PARENT_NODE_ID,
CONSTRUCT_STATEMENTS)),
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAO.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAO.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAO.java
index 8675b80..5ba7383 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAO.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAO.java
@@ -25,7 +25,9 @@ import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.Collection;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.fluo.api.client.SnapshotBase;
@@ -34,6 +36,9 @@ import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.Column;
import org.apache.rya.indexing.pcj.fluo.app.ConstructGraph;
import org.apache.rya.indexing.pcj.fluo.app.ConstructGraphSerializer;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.ExportStrategy;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.QueryType;
import org.apache.rya.indexing.pcj.fluo.app.NodeType;
import org.apache.rya.indexing.pcj.fluo.app.query.AggregationMetadata.AggregationElement;
import org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata.JoinType;
@@ -42,7 +47,6 @@ import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
import edu.umd.cs.findbugs.annotations.NonNull;
@@ -64,10 +68,14 @@ public class FluoQueryMetadataDAO {
requireNonNull(tx);
requireNonNull(metadata);
+ Joiner joiner = Joiner.on(IncrementalUpdateConstants.VAR_DELIM);
+
final String rowId = metadata.getNodeId();
tx.set(rowId, FluoQueryColumns.QUERY_NODE_ID, rowId);
tx.set(rowId, FluoQueryColumns.QUERY_VARIABLE_ORDER, metadata.getVariableOrder().toString());
tx.set(rowId, FluoQueryColumns.QUERY_SPARQL, metadata.getSparql() );
+ tx.set(rowId, FluoQueryColumns.QUERY_EXPORT_STRATEGIES, joiner.join(metadata.getExportStrategies()));
+ tx.set(rowId, FluoQueryColumns.QUERY_TYPE, metadata.getQueryType().toString());
tx.set(rowId, FluoQueryColumns.QUERY_CHILD_NODE_ID, metadata.getChildNodeId() );
}
@@ -91,6 +99,8 @@ public class FluoQueryMetadataDAO {
final Map<Column, String> values = sx.gets(rowId,
FluoQueryColumns.QUERY_VARIABLE_ORDER,
FluoQueryColumns.QUERY_SPARQL,
+ FluoQueryColumns.QUERY_TYPE,
+ FluoQueryColumns.QUERY_EXPORT_STRATEGIES,
FluoQueryColumns.QUERY_CHILD_NODE_ID);
// Return an object holding them.
@@ -99,13 +109,81 @@ public class FluoQueryMetadataDAO {
final String sparql = values.get(FluoQueryColumns.QUERY_SPARQL);
final String childNodeId = values.get(FluoQueryColumns.QUERY_CHILD_NODE_ID);
+ final String queryType = values.get(FluoQueryColumns.QUERY_TYPE);
+ final String[] exportStrategies = values.get(FluoQueryColumns.QUERY_EXPORT_STRATEGIES).split(IncrementalUpdateConstants.VAR_DELIM);
+
+ Set<ExportStrategy> strategies = new HashSet<>();
+ for(String strategy: exportStrategies) {
+ strategies.add(ExportStrategy.valueOf(strategy));
+ }
return QueryMetadata.builder(nodeId)
- .setVariableOrder( varOrder )
+ .setVarOrder( varOrder )
.setSparql( sparql )
+ .setExportStrategies(strategies)
+ .setQueryType(QueryType.valueOf(queryType))
.setChildNodeId( childNodeId );
}
+
+
+ /**
+ * Write an instance of {@link ProjectionMetadata} to the Fluo table.
+ *
+ * @param tx - The transaction that will be used to commit the metadata. (not null)
+ * @param metadata - The Query node metadata that will be written to the table. (not null)
+ */
+ public void write(final TransactionBase tx, final ProjectionMetadata metadata) {
+ requireNonNull(tx);
+ requireNonNull(metadata);
+
+ final String rowId = metadata.getNodeId();
+ tx.set(rowId, FluoQueryColumns.PROJECTION_NODE_ID, rowId);
+ tx.set(rowId, FluoQueryColumns.PROJECTION_VARIABLE_ORDER, metadata.getVariableOrder().toString());
+ tx.set(rowId, FluoQueryColumns.PROJECTION_PROJECTED_VARS, metadata.getProjectedVars().toString());
+ tx.set(rowId, FluoQueryColumns.PROJECTION_PARENT_NODE_ID, metadata.getParentNodeId());
+ tx.set(rowId, FluoQueryColumns.PROJECTION_CHILD_NODE_ID, metadata.getChildNodeId() );
+ }
+
+ /**
+ * Read an instance of {@link ProjectionMetadata} from the Fluo table.
+ *
+ * @param sx - The snapshot that will be used to read the metadata . (not null)
+ * @param nodeId - The nodeId of the Projection node that will be read. (not null)
+ * @return The {@link ProjectionMetadata} that was read from the table.
+ */
+ public ProjectionMetadata readProjectionMetadata(final SnapshotBase sx, final String nodeId) {
+ return readProjectionMetadataBuilder(sx, nodeId).build();
+ }
+
+ private ProjectionMetadata.Builder readProjectionMetadataBuilder(final SnapshotBase sx, final String nodeId) {
+ requireNonNull(sx);
+ requireNonNull(nodeId);
+
+ // Fetch the values from the Fluo table.
+ final String rowId = nodeId;
+ final Map<Column, String> values = sx.gets(rowId,
+ FluoQueryColumns.PROJECTION_VARIABLE_ORDER,
+ FluoQueryColumns.PROJECTION_PROJECTED_VARS,
+ FluoQueryColumns.PROJECTION_PARENT_NODE_ID,
+ FluoQueryColumns.PROJECTION_CHILD_NODE_ID);
+ // Return an object holding them.
+ final String varOrderString = values.get(FluoQueryColumns.PROJECTION_VARIABLE_ORDER);
+ final String projectedVarString = values.get(FluoQueryColumns.PROJECTION_PROJECTED_VARS);
+ final VariableOrder varOrder = new VariableOrder(varOrderString);
+ final VariableOrder projectedVars = new VariableOrder(projectedVarString);
+ final String childNodeId = values.get(FluoQueryColumns.PROJECTION_CHILD_NODE_ID);
+ final String parentNodeId = values.get(FluoQueryColumns.PROJECTION_PARENT_NODE_ID);
+
+
+ return ProjectionMetadata.builder(nodeId)
+ .setVarOrder( varOrder )
+ .setProjectedVars(projectedVars)
+ .setParentNodeId(parentNodeId)
+ .setChildNodeId( childNodeId );
+ }
+
+
/**
* Write an instance of {@link ConstructQueryMetadata} to the Fluo table.
*
@@ -120,7 +198,7 @@ public class FluoQueryMetadataDAO {
tx.set(rowId, FluoQueryColumns.CONSTRUCT_NODE_ID, rowId);
tx.set(rowId, FluoQueryColumns.CONSTRUCT_VARIABLE_ORDER, metadata.getVariableOrder().toString());
tx.set(rowId, FluoQueryColumns.CONSTRUCT_CHILD_NODE_ID, metadata.getChildNodeId() );
- tx.set(rowId, FluoQueryColumns.CONSTRUCT_SPARQL, metadata.getSparql());
+ tx.set(rowId, FluoQueryColumns.CONSTRUCT_PARENT_NODE_ID, metadata.getParentNodeId() );
tx.set(rowId, FluoQueryColumns.CONSTRUCT_GRAPH, ConstructGraphSerializer.toConstructString(metadata.getConstructGraph()));
}
@@ -143,18 +221,22 @@ public class FluoQueryMetadataDAO {
final String rowId = nodeId;
final Map<Column, String> values = sx.gets(rowId,
FluoQueryColumns.CONSTRUCT_GRAPH,
- FluoQueryColumns.CONSTRUCT_SPARQL,
- FluoQueryColumns.CONSTRUCT_CHILD_NODE_ID);
+ FluoQueryColumns.CONSTRUCT_CHILD_NODE_ID,
+ FluoQueryColumns.CONSTRUCT_PARENT_NODE_ID,
+ FluoQueryColumns.CONSTRUCT_VARIABLE_ORDER);
final String graphString = values.get(FluoQueryColumns.CONSTRUCT_GRAPH);
final ConstructGraph graph = ConstructGraphSerializer.toConstructGraph(graphString);
final String childNodeId = values.get(FluoQueryColumns.CONSTRUCT_CHILD_NODE_ID);
- final String sparql = values.get(FluoQueryColumns.CONSTRUCT_SPARQL);
+ final String parentNodeId = values.get(FluoQueryColumns.CONSTRUCT_PARENT_NODE_ID);
+ final String varOrderString = values.get(FluoQueryColumns.CONSTRUCT_VARIABLE_ORDER);
+
return ConstructQueryMetadata.builder()
.setNodeId(nodeId)
+ .setParentNodeId(parentNodeId)
.setConstructGraph(graph)
- .setSparql(sparql)
+ .setVarOrder(new VariableOrder(varOrderString))
.setChildNodeId(childNodeId);
}
@@ -342,7 +424,7 @@ public class FluoQueryMetadataDAO {
final String rightChildNodeId = values.get(FluoQueryColumns.JOIN_RIGHT_CHILD_NODE_ID);
return JoinMetadata.builder(nodeId)
- .setVariableOrder(varOrder)
+ .setVarOrder(varOrder)
.setJoinType(joinType)
.setParentNodeId(parentNodeId)
.setLeftChildNodeId(leftChildNodeId)
@@ -477,7 +559,7 @@ public class FluoQueryMetadataDAO {
}
final AggregationMetadata.Builder builder = AggregationMetadata.builder(nodeId)
- .setVariableOrder(varOrder)
+ .setVarOrder(varOrder)
.setParentNodeId(parentNodeId)
.setChildNodeId(childNodeId)
.setGroupByVariableOrder(groupByVars);
@@ -498,27 +580,28 @@ public class FluoQueryMetadataDAO {
public void write(final TransactionBase tx, final FluoQuery query) {
requireNonNull(tx);
requireNonNull(query);
+
+ QueryMetadata queryMetadata = query.getQueryMetadata();
+ final String sparql = queryMetadata.getSparql();
+ final String queryId = queryMetadata.getNodeId();
+ final String pcjId = queryMetadata.getExportId();
+
+ // The results of the query are eventually exported to an instance
+ // of Rya, so store the Rya ID for the PCJ.
+ tx.set(queryId, FluoQueryColumns.RYA_PCJ_ID, pcjId);
+ tx.set(pcjId, FluoQueryColumns.PCJ_ID_QUERY_ID, queryId);
+ tx.set(Bytes.of(sparql), FluoQueryColumns.QUERY_ID, Bytes.of(queryId));
+ write(tx, queryMetadata);
// Write the rest of the metadata objects.
- switch (query.getQueryType()) {
- case Construct:
+
+ if (query.getQueryType() == QueryType.Construct) {
ConstructQueryMetadata constructMetadata = query.getConstructQueryMetadata().get();
- // Store the Query ID so that it may be looked up from the original
- // SPARQL string.
- final String constructSparql = constructMetadata.getSparql();
- final String constructQueryId = constructMetadata.getNodeId();
- tx.set(Bytes.of(constructSparql), FluoQueryColumns.QUERY_ID, Bytes.of(constructQueryId));
write(tx, constructMetadata);
- break;
- case Projection:
- QueryMetadata queryMetadata = query.getQueryMetadata().get();
- // Store the Query ID so that it may be looked up from the original
- // SPARQL string.
- final String sparql = queryMetadata.getSparql();
- final String queryId = queryMetadata.getNodeId();
- tx.set(Bytes.of(sparql), FluoQueryColumns.QUERY_ID, Bytes.of(queryId));
- write(tx, queryMetadata);
- break;
+ }
+
+ for(final ProjectionMetadata projection : query.getProjectionMetadata()) {
+ write(tx, projection);
}
Optional<PeriodicQueryMetadata> periodicMetadata = query.getPeriodicQueryMetadata();
@@ -569,16 +652,23 @@ public class FluoQueryMetadataDAO {
case QUERY:
// Add this node's metadata.
final QueryMetadata.Builder queryBuilder = readQueryMetadataBuilder(sx, childNodeId);
- Preconditions.checkArgument(!builder.getQueryBuilder().isPresent());
builder.setQueryMetadata(queryBuilder);
// Add it's child's metadata.
addChildMetadata(sx, builder, queryBuilder.build().getChildNodeId());
break;
-
+
+ case PROJECTION:
+ //Add this node's metadata
+ final ProjectionMetadata.Builder projectionBuilder = readProjectionMetadataBuilder(sx, childNodeId);
+ builder.addProjectionBuilder(projectionBuilder);
+
+ //Add it's child's metadata
+ addChildMetadata(sx, builder, projectionBuilder.build().getChildNodeId());
+ break;
+
case CONSTRUCT:
final ConstructQueryMetadata.Builder constructBuilder = readConstructQueryMetadataBuilder(sx, childNodeId);
- Preconditions.checkArgument(!builder.getQueryBuilder().isPresent());
builder.setConstructQueryMetadata(constructBuilder);
// Add it's child's metadata.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/JoinMetadata.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/JoinMetadata.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/JoinMetadata.java
index 7bad9a7..aa79daf 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/JoinMetadata.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/JoinMetadata.java
@@ -163,7 +163,7 @@ public class JoinMetadata extends CommonNodeMetadata {
* Builds instances of {@link JoinMetadata}.
*/
@DefaultAnnotation(NonNull.class)
- public static final class Builder {
+ public static final class Builder implements CommonNodeMetadata.Builder {
private final String nodeId;
private VariableOrder varOrder;
@@ -194,11 +194,16 @@ public class JoinMetadata extends CommonNodeMetadata {
* @param varOrder - The variable order of the binding sets that are emitted by this node.
* @return This builder so that method invocation could be chained.
*/
- public Builder setVariableOrder(@Nullable final VariableOrder varOrder) {
+ public Builder setVarOrder(@Nullable final VariableOrder varOrder) {
this.varOrder = varOrder;
return this;
}
+ @Override
+ public VariableOrder getVariableOrder() {
+ return varOrder;
+ }
+
/**
* Sets the node id of this node's parent.
*
@@ -242,6 +247,14 @@ public class JoinMetadata extends CommonNodeMetadata {
this.rightChildNodeId = rightChildNodeId;
return this;
}
+
+ public String getLeftChildNodeId() {
+ return leftChildNodeId;
+ }
+
+ public String getRightChildNodeId() {
+ return rightChildNodeId;
+ }
/**
* @return An instance of {@link JoinMetadata} built using this builder's values.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/PeriodicQueryMetadata.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/PeriodicQueryMetadata.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/PeriodicQueryMetadata.java
index 33253f2..ae4b10e 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/PeriodicQueryMetadata.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/PeriodicQueryMetadata.java
@@ -166,7 +166,7 @@ public class PeriodicQueryMetadata extends CommonNodeMetadata {
/**
* Builder for chaining method calls to construct an instance of PeriodicQueryMetadata.
*/
- public static class Builder {
+ public static class Builder implements CommonNodeMetadata.Builder {
private String nodeId;
private VariableOrder varOrder;
@@ -200,11 +200,12 @@ public class PeriodicQueryMetadata extends CommonNodeMetadata {
return this;
}
+
/**
* Returns {@link VariableOrder}
* @return VariableOrder that indicates order that results are written in
*/
- public VariableOrder getVarOrder() {
+ public VariableOrder getVariableOrder() {
return varOrder;
}
@@ -235,6 +236,10 @@ public class PeriodicQueryMetadata extends CommonNodeMetadata {
return this;
}
+ public String getChildNodeId() {
+ return childNodeId;
+ }
+
/**
* Sets window size for periodic query
* @param windowSize
[5/5] incubator-rya git commit: RYA-282-Nested-Query. Closes #192.
Posted by ca...@apache.org.
RYA-282-Nested-Query. Closes #192.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rya/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rya/commit/e387818b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rya/tree/e387818b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rya/diff/e387818b
Branch: refs/heads/master
Commit: e387818ba22b07a07432d62eea172da6c33d793f
Parents: 6ce0b00
Author: Caleb Meier <ca...@parsons.com>
Authored: Thu Jul 20 06:57:38 2017 -0700
Committer: Caleb Meier <ca...@parsons.com>
Committed: Fri Aug 18 11:50:36 2017 -0700
----------------------------------------------------------------------
.../api/client/accumulo/AccumuloCreatePCJ.java | 3 +-
.../api/client/accumulo/AccumuloDeletePCJ.java | 4 +-
.../client/accumulo/AccumuloCreatePCJIT.java | 1 +
.../rya/api/client/accumulo/FluoITBase.java | 4 +
.../indexing/pcj/fluo/api/CreateFluoPcj.java | 385 ++++++++++++++
.../rya/indexing/pcj/fluo/api/CreatePcj.java | 450 ----------------
.../indexing/pcj/fluo/api/DeleteFluoPcj.java | 312 +++++++++++
.../rya/indexing/pcj/fluo/api/DeletePcj.java | 305 -----------
.../pcj/fluo/app/ConstructProjection.java | 4 -
.../fluo/app/ConstructQueryResultUpdater.java | 46 +-
.../pcj/fluo/app/FilterResultUpdater.java | 23 +-
.../pcj/fluo/app/FluoStringConverter.java | 2 -
.../fluo/app/IncrementalUpdateConstants.java | 4 +
.../pcj/fluo/app/JoinResultUpdater.java | 31 +-
.../rya/indexing/pcj/fluo/app/NodeType.java | 57 +-
.../pcj/fluo/app/PeriodicQueryUpdater.java | 1 -
.../pcj/fluo/app/ProjectionResultUpdater.java | 89 ++++
.../pcj/fluo/app/QueryResultUpdater.java | 20 +-
.../export/kafka/KafkaRyaSubGraphExporter.java | 2 -
.../fluo/app/observers/BindingSetUpdater.java | 14 +-
.../fluo/app/observers/ProjectionObserver.java | 65 +++
.../fluo/app/observers/QueryResultObserver.java | 2 +-
.../pcj/fluo/app/query/AggregationMetadata.java | 8 +-
.../pcj/fluo/app/query/CommonNodeMetadata.java | 14 +
.../fluo/app/query/ConstructQueryMetadata.java | 94 ++--
.../pcj/fluo/app/query/FilterMetadata.java | 11 +-
.../indexing/pcj/fluo/app/query/FluoQuery.java | 234 ++++++---
.../pcj/fluo/app/query/FluoQueryColumns.java | 45 +-
.../fluo/app/query/FluoQueryMetadataDAO.java | 148 ++++--
.../pcj/fluo/app/query/JoinMetadata.java | 17 +-
.../fluo/app/query/PeriodicQueryMetadata.java | 9 +-
.../pcj/fluo/app/query/ProjectionMetadata.java | 236 +++++++++
.../fluo/app/query/QueryBuilderVisitorBase.java | 119 +++++
.../pcj/fluo/app/query/QueryMetadata.java | 98 +++-
.../app/query/QueryMetadataVisitorBase.java | 113 ++++
.../fluo/app/query/SparqlFluoQueryBuilder.java | 444 +++++++++++-----
.../app/query/StatementPatternMetadata.java | 7 +-
.../pcj/fluo/app/util/FluoQueryUtils.java | 62 +++
.../pcj/fluo/app/util/PeriodicQueryUtil.java | 76 +--
.../app/util/VariableOrderUpdateVisitor.java | 166 ++++++
.../fluo/app/query/PeriodicQueryUtilTest.java | 10 +-
.../fluo/app/query/QueryBuilderVisitorTest.java | 105 ++++
.../app/query/QueryMetadataVisitorTest.java | 109 ++++
.../fluo/client/command/NewQueryCommand.java | 4 +-
.../fluo/client/util/QueryReportRenderer.java | 41 +-
.../pcj/fluo/demo/FluoAndHistoricPcjsDemo.java | 4 +-
.../pcj/fluo/ConstructGraphTestUtils.java | 15 +-
.../indexing/pcj/fluo/api/GetPcjMetadataIT.java | 4 +-
.../indexing/pcj/fluo/api/GetQueryReportIT.java | 4 +-
.../fluo/app/query/FluoQueryMetadataDAOIT.java | 163 +++++-
.../pcj/fluo/integration/BatchDeleteIT.java | 8 +-
.../pcj/fluo/integration/CreateDeleteIT.java | 10 +-
.../indexing/pcj/fluo/integration/InputIT.java | 10 +-
.../pcj/fluo/integration/KafkaExportIT.java | 163 +++++-
.../integration/KafkaRyaSubGraphExportIT.java | 86 ++-
.../indexing/pcj/fluo/integration/QueryIT.java | 517 ++++++++++++-------
.../pcj/fluo/integration/RyaExportIT.java | 4 +-
.../RyaInputIncrementalUpdateIT.java | 8 +-
.../pcj/fluo/integration/StreamingTestIT.java | 4 +-
.../HistoricStreamingVisibilityIT.java | 4 +-
.../pcj/fluo/visibility/PcjVisibilityIT.java | 4 +-
.../pcj/fluo/test/base/KafkaExportITBase.java | 4 +-
.../rya/pcj/fluo/test/base/RyaExportITBase.java | 2 +
.../PeriodicNotificationProviderIT.java | 9 +-
.../notification/api/CreatePeriodicQuery.java | 10 +-
.../recovery/PeriodicNotificationProvider.java | 9 +-
66 files changed, 3569 insertions(+), 1467 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloCreatePCJ.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloCreatePCJ.java b/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloCreatePCJ.java
index 3fe1042..644189a 100644
--- a/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloCreatePCJ.java
+++ b/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloCreatePCJ.java
@@ -38,6 +38,7 @@ import org.apache.rya.api.instance.RyaDetailsUpdater;
import org.apache.rya.api.instance.RyaDetailsUpdater.RyaDetailsMutator;
import org.apache.rya.api.instance.RyaDetailsUpdater.RyaDetailsMutator.CouldNotApplyMutationException;
import org.apache.rya.api.persist.RyaDAOException;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
import org.apache.rya.indexing.pcj.storage.PcjException;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
@@ -145,7 +146,7 @@ public class AccumuloCreatePCJ extends AccumuloCommand implements CreatePCJ {
cd.getZookeepers(),
fluoAppName);) {
// Initialize the PCJ within the Fluo application.
- final org.apache.rya.indexing.pcj.fluo.api.CreatePcj fluoCreatePcj = new org.apache.rya.indexing.pcj.fluo.api.CreatePcj();
+ final CreateFluoPcj fluoCreatePcj = new CreateFluoPcj();
fluoCreatePcj.withRyaIntegration(pcjId, pcjStorage, fluoClient, getConnector(), ryaInstance);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloDeletePCJ.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloDeletePCJ.java b/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloDeletePCJ.java
index 96e6d58..eb2b2d7 100644
--- a/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloDeletePCJ.java
+++ b/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloDeletePCJ.java
@@ -31,7 +31,7 @@ import org.apache.rya.api.instance.RyaDetails.PCJIndexDetails;
import org.apache.rya.api.instance.RyaDetails.PCJIndexDetails.FluoDetails;
import org.apache.rya.api.instance.RyaDetails.PCJIndexDetails.PCJDetails;
import org.apache.rya.api.instance.RyaDetails.PCJIndexDetails.PCJDetails.PCJUpdateStrategy;
-import org.apache.rya.indexing.pcj.fluo.api.DeletePcj;
+import org.apache.rya.indexing.pcj.fluo.api.DeleteFluoPcj;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException;
import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage;
@@ -123,7 +123,7 @@ public class AccumuloDeletePCJ extends AccumuloCommand implements DeletePCJ {
cd.getZookeepers(),
fluoAppName)) {
// Delete the PCJ from the Fluo App.
- new DeletePcj(1000).deletePcj(fluoClient, pcjId);
+ new DeleteFluoPcj(1000).deletePcj(fluoClient, pcjId);
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloCreatePCJIT.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloCreatePCJIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloCreatePCJIT.java
index 9bbf01f..3463a02 100644
--- a/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloCreatePCJIT.java
+++ b/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloCreatePCJIT.java
@@ -34,6 +34,7 @@ import org.apache.rya.api.instance.RyaDetails;
import org.apache.rya.api.instance.RyaDetails.PCJIndexDetails.PCJDetails;
import org.apache.rya.api.instance.RyaDetails.PCJIndexDetails.PCJDetails.PCJUpdateStrategy;
import org.apache.rya.indexing.pcj.fluo.api.ListQueryIds;
+import org.apache.rya.indexing.pcj.fluo.app.IncUpdateDAO;
import org.apache.rya.indexing.pcj.storage.PcjMetadata;
import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage;
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/FluoITBase.java
----------------------------------------------------------------------
diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/FluoITBase.java b/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/FluoITBase.java
index 2e16412..113b397 100644
--- a/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/FluoITBase.java
+++ b/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/FluoITBase.java
@@ -39,8 +39,10 @@ import org.apache.rya.accumulo.MiniAccumuloClusterInstance;
import org.apache.rya.accumulo.MiniAccumuloSingleton;
import org.apache.rya.accumulo.RyaTestInstanceRule;
import org.apache.rya.indexing.pcj.fluo.app.export.rya.RyaExportParameters;
+import org.apache.rya.indexing.pcj.fluo.app.observers.ConstructQueryResultObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.FilterObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.JoinObserver;
+import org.apache.rya.indexing.pcj.fluo.app.observers.ProjectionObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.QueryResultObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.StatementPatternObserver;
import org.apache.rya.indexing.pcj.fluo.app.observers.TripleObserver;
@@ -219,6 +221,8 @@ public abstract class FluoITBase {
observers.add(new ObserverSpecification(StatementPatternObserver.class.getName()));
observers.add(new ObserverSpecification(JoinObserver.class.getName()));
observers.add(new ObserverSpecification(FilterObserver.class.getName()));
+ observers.add(new ObserverSpecification(ProjectionObserver.class.getName()));
+ observers.add(new ObserverSpecification(ConstructQueryResultObserver.class.getName()));
// Provide export parameters child test classes may provide to the
// export observer.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreateFluoPcj.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreateFluoPcj.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreateFluoPcj.java
new file mode 100644
index 0000000..e450960
--- /dev/null
+++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreateFluoPcj.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.fluo.api;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.client.Transaction;
+import org.apache.log4j.Logger;
+import org.apache.rya.accumulo.AccumuloRdfConfiguration;
+import org.apache.rya.accumulo.query.AccumuloRyaQueryEngine;
+import org.apache.rya.api.domain.RyaStatement;
+import org.apache.rya.api.domain.RyaType;
+import org.apache.rya.api.domain.RyaURI;
+import org.apache.rya.api.persist.RyaDAOException;
+import org.apache.rya.api.persist.query.BatchRyaQuery;
+import org.apache.rya.api.resolver.RdfToRyaConversions;
+import org.apache.rya.indexing.pcj.fluo.app.FluoStringConverter;
+import org.apache.rya.indexing.pcj.fluo.app.NodeType;
+import org.apache.rya.indexing.pcj.fluo.app.query.FluoQuery;
+import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns;
+import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryMetadataDAO;
+import org.apache.rya.indexing.pcj.fluo.app.query.QueryMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.SparqlFluoQueryBuilder;
+import org.apache.rya.indexing.pcj.fluo.app.query.StatementPatternMetadata;
+import org.apache.rya.indexing.pcj.storage.PcjException;
+import org.apache.rya.indexing.pcj.storage.PcjMetadata;
+import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
+import org.calrissian.mango.collect.CloseableIterable;
+import org.openrdf.model.Resource;
+import org.openrdf.model.URI;
+import org.openrdf.model.Value;
+import org.openrdf.query.MalformedQueryException;
+import org.openrdf.query.algebra.StatementPattern;
+
+import com.google.common.base.Preconditions;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
+
+/**
+ * Sets up a new Pre Computed Join (PCJ) in Fluo from a SPARQL query.
+ * <p>
+ * This is a two phase process.
+ * <ol>
+ * <li>Setup metadata about each node of the query using a single Fluo transaction. </li>
+ * <li>Scan Rya for binding sets that match each Statement Pattern from the query
+ * and use a separate Fluo transaction for each batch that is inserted. This
+ * ensure historic triples will be included in the query's results.</li>
+ * </ol>
+ * After the first step is finished, any new Triples that are added to the Fluo
+ * application will be matched against statement patterns, the final results
+ * will percolate to the top of the query, and those results will be exported to
+ * Rya's query system.
+ */
+@DefaultAnnotation(NonNull.class)
+public class CreateFluoPcj {
+ private static final Logger log = Logger.getLogger(CreateFluoPcj.class);
+
+ /**
+ * The default Statement Pattern batch insert size is 1000.
+ */
+ private static final int DEFAULT_SP_INSERT_BATCH_SIZE = 1000;
+
+ /**
+ * The maximum number of binding sets that will be inserted into each Statement
+ * Pattern's result set per Fluo transaction.
+ */
+ private final int spInsertBatchSize;
+
+ /**
+ * Constructs an instance of {@link CreateFluoPcj} that uses
+ * {@link #DEFAULT_SP_INSERT_BATCH_SIZE} as the default batch insert size.
+ */
+ public CreateFluoPcj() {
+ this(DEFAULT_SP_INSERT_BATCH_SIZE);
+ }
+
+ /**
+ * Constructs an instance of {@link CreateFluoPcj}.
+ *
+ * @param spInsertBatchSize - The maximum number of binding sets that will be
+ * inserted into each Statement Pattern's result set per Fluo transaction.
+ */
+ public CreateFluoPcj(final int spInsertBatchSize) {
+ checkArgument(spInsertBatchSize > 0, "The SP insert batch size '" + spInsertBatchSize + "' must be greater than 0.");
+ this.spInsertBatchSize = spInsertBatchSize;
+ }
+
+
+ /**
+ * Tells the Fluo PCJ Updater application to maintain a new PCJ. This method
+ * creates the FluoQuery (metadata) inside of Fluo so that results can be incrementally generated
+ * inside of Fluo. This method assumes that the user will export the results to Kafka or
+ * some other external resource. The export id is equivalent to the queryId that is returned,
+ * which is in contrast to the other createPcj methods in this class which accept an external pcjId
+ * that is used to identify the Accumulo table or Kafka topic for exporting results.
+ *
+ * @param sparql - sparql query String to be registered with Fluo
+ * @param fluo - A connection to the Fluo application that updates the PCJ index. (not null)
+ * @return The metadata that was written to the Fluo application for the PCJ.
+ * @throws MalformedQueryException The SPARQL query stored for the {@code pcjId} is malformed.
+ * @throws PcjException The PCJ Metadata for {@code pcjId} could not be read from {@code pcjStorage}.
+ */
+ public FluoQuery createPcj(String sparql, FluoClient fluo) throws MalformedQueryException {
+ Preconditions.checkNotNull(sparql);
+ Preconditions.checkNotNull(fluo);
+
+ String pcjId = UUID.randomUUID().toString().replaceAll("-", "");
+ return createPcj(pcjId, sparql, fluo);
+ }
+
+ /**
+ * Tells the Fluo PCJ Updater application to maintain a new PCJ. This method provides
+ * no guarantees that a PCJ with the given pcjId exists outside of Fluo. This method merely
+ * creates the FluoQuery (metadata) inside of Fluo so that results and be incrementally generated
+ * inside of Fluo. This method assumes that the user will export the results to Kafka or
+ * some other external resource.
+ *
+ * @param pcjId - Identifies the PCJ that will be updated by the Fluo app. (not null)
+ * @param sparql - sparql query String to be registered with Fluo
+ * @param fluo - A connection to the Fluo application that updates the PCJ index. (not null)
+ * @return The metadata that was written to the Fluo application for the PCJ.
+ * @throws PcjException The PCJ Metadata for {@code pcjId} could not be read from {@code pcjStorage}.
+ */
+ public FluoQuery createPcj(
+ final String pcjId,
+ final String sparql,
+ final FluoClient fluo) throws MalformedQueryException {
+ requireNonNull(pcjId);
+ requireNonNull(sparql);
+ requireNonNull(fluo);
+
+ FluoQuery fluoQuery = makeFluoQuery(sparql, pcjId);
+ writeFluoQuery(fluo, fluoQuery, pcjId);
+
+ return fluoQuery;
+ }
+
+ private FluoQuery makeFluoQuery(String sparql, String pcjId) throws MalformedQueryException {
+
+ String queryId = NodeType.generateNewIdForType(NodeType.QUERY, pcjId);
+
+ SparqlFluoQueryBuilder builder = new SparqlFluoQueryBuilder();
+ builder.setFluoQueryId(queryId);
+ builder.setSparql(sparql);
+
+ return builder.build();
+ }
+
+ private void writeFluoQuery(FluoClient fluo, FluoQuery fluoQuery, String pcjId) {
+ try (Transaction tx = fluo.newTransaction()) {
+ // Write the query's structure to Fluo.
+ new FluoQueryMetadataDAO().write(tx, fluoQuery);
+
+ // Flush the changes to Fluo.
+ tx.commit();
+ }
+ }
+
+
+ /**
+ * Tells the Fluo PCJ Updater application to maintain a new PCJ. The method takes in an
+ * instance of {@link PrecomputedJoinStorage} to verify that a PCJ with the given pcjId exists.
+ *
+ * @param pcjId - Identifies the PCJ that will be updated by the Fluo app. (not null)
+ * @param pcjStorage - Provides access to the PCJ index. (not null)
+ * @param fluo - A connection to the Fluo application that updates the PCJ index. (not null)
+ * @return The metadata that was written to the Fluo application for the PCJ.
+ * @throws MalformedQueryException The SPARQL query stored for the {@code pcjId} is malformed.
+ * @throws PcjException The PCJ Metadata for {@code pcjId} could not be read from {@code pcjStorage}.
+ */
+ public FluoQuery createPcj(
+ final String pcjId,
+ final PrecomputedJoinStorage pcjStorage,
+ final FluoClient fluo) throws MalformedQueryException, PcjException {
+ requireNonNull(pcjId);
+ requireNonNull(pcjStorage);
+ requireNonNull(fluo);
+
+ // Parse the query's structure for the metadata that will be written to fluo.
+ final PcjMetadata pcjMetadata = pcjStorage.getPcjMetadata(pcjId);
+ final String sparql = pcjMetadata.getSparql();
+ return createPcj(pcjId, sparql, fluo);
+ }
+
+ /**
+ * Tells the Fluo PCJ Updater application to maintain a new PCJ.
+ * <p>
+ * This call scans Rya for Statement Pattern matches and inserts them into
+ * the Fluo application. This method does not verify that a PcjTable with the
+ * the given pcjId actually exists. It is assumed that results for any query registered
+ * using this method will be exported to Kafka or some other external service.
+ *
+ * @param pcjId - Identifies the PCJ that will be updated by the Fluo app. (not null)
+ * @param sparql - sparql query that will registered with Fluo. (not null)
+ * @param fluo - A connection to the Fluo application that updates the PCJ index. (not null)
+ * @param queryEngine - QueryEngine for a given Rya Instance, (not null)
+ * @return The Fluo application's Query ID of the query that was created.
+ * @throws MalformedQueryException The SPARQL query stored for the {@code pcjId} is malformed.
+ * @throws PcjException The PCJ Metadata for {@code pcjId} could not be read from {@code pcjStorage}.
+ * @throws RyaDAOException Historic PCJ results could not be loaded because of a problem with {@code rya}.
+ */
+ public String withRyaIntegration(
+ final String pcjId,
+ final String sparql,
+ final FluoClient fluo,
+ final Connector accumulo,
+ final String ryaInstance ) throws MalformedQueryException, PcjException, RyaDAOException {
+ requireNonNull(pcjId);
+ requireNonNull(sparql);
+ requireNonNull(fluo);
+ requireNonNull(accumulo);
+ requireNonNull(ryaInstance);
+
+
+ // Write the SPARQL query's structure to the Fluo Application.
+ final FluoQuery fluoQuery = createPcj(pcjId, sparql, fluo);
+ //import results already ingested into Rya that match query
+ importHistoricResultsIntoFluo(fluo, fluoQuery, accumulo, ryaInstance);
+ // return queryId to the caller for later monitoring from the export.
+ return fluoQuery.getQueryMetadata().getNodeId();
+ }
+
+
+ /**
+ * Tells the Fluo PCJ Updater application to maintain a new PCJ.
+ * <p>
+ * This call scans Rya for Statement Pattern matches and inserts them into
+ * the Fluo application. The Fluo application will then maintain the intermediate
+ * results as new triples are inserted and export any new query results to the
+ * {@code pcjId} within the provided {@code pcjStorage}. This method requires that a
+ * PCJ table already exist for the query corresponding to the pcjId. Results will be exported
+ * to this table.
+ *
+ * @param pcjId - Identifies the PCJ that will be updated by the Fluo app. (not null)
+ * @param pcjStorage - Provides access to the PCJ index. (not null)
+ * @param fluo - A connection to the Fluo application that updates the PCJ index. (not null)
+ * @param queryEngine - QueryEngine for a given Rya Instance, (not null)
+ * @return The Fluo application's Query ID of the query that was created.
+ * @throws MalformedQueryException The SPARQL query stored for the {@code pcjId} is malformed.
+ * @throws PcjException The PCJ Metadata for {@code pcjId} could not be read from {@code pcjStorage}.
+ * @throws RyaDAOException Historic PCJ results could not be loaded because of a problem with {@code rya}.
+ */
+ public String withRyaIntegration(
+ final String pcjId,
+ final PrecomputedJoinStorage pcjStorage,
+ final FluoClient fluo,
+ final Connector accumulo,
+ final String ryaInstance ) throws MalformedQueryException, PcjException, RyaDAOException {
+ requireNonNull(pcjId);
+ requireNonNull(pcjStorage);
+ requireNonNull(fluo);
+ requireNonNull(accumulo);
+ requireNonNull(ryaInstance);
+
+ // Parse the query's structure for the metadata that will be written to fluo.
+ final PcjMetadata pcjMetadata = pcjStorage.getPcjMetadata(pcjId);
+ final String sparql = pcjMetadata.getSparql();
+
+ return withRyaIntegration(pcjId, sparql, fluo, accumulo, ryaInstance);
+ }
+
+ private void importHistoricResultsIntoFluo(FluoClient fluo, FluoQuery fluoQuery, Connector accumulo, String ryaInstance)
+ throws RyaDAOException {
+ // Reuse the same set object while performing batch inserts.
+ final Set<RyaStatement> queryBatch = new HashSet<>();
+
+ // Iterate through each of the statement patterns and insert their
+ // historic matches into Fluo.
+ for (final StatementPatternMetadata patternMetadata : fluoQuery.getStatementPatternMetadata()) {
+ // Get an iterator over all of the binding sets that match the
+ // statement pattern.
+ final StatementPattern pattern = FluoStringConverter.toStatementPattern(patternMetadata.getStatementPattern());
+ queryBatch.add(spToRyaStatement(pattern));
+ }
+
+ // Create AccumuloRyaQueryEngine to query for historic results
+ final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration();
+ conf.setTablePrefix(ryaInstance);
+ conf.setAuths(getAuths(accumulo));
+
+ try (final AccumuloRyaQueryEngine queryEngine = new AccumuloRyaQueryEngine(accumulo, conf);
+ CloseableIterable<RyaStatement> queryIterable = queryEngine.query(new BatchRyaQuery(queryBatch))) {
+ final Set<RyaStatement> triplesBatch = new HashSet<>();
+
+ // Insert batches of the binding sets into Fluo.
+ for (final RyaStatement ryaStatement : queryIterable) {
+ if (triplesBatch.size() == spInsertBatchSize) {
+ writeBatch(fluo, triplesBatch);
+ triplesBatch.clear();
+ }
+
+ triplesBatch.add(ryaStatement);
+ }
+
+ if (!triplesBatch.isEmpty()) {
+ writeBatch(fluo, triplesBatch);
+ triplesBatch.clear();
+ }
+ } catch (final IOException e) {
+ log.warn("Ignoring IOException thrown while closing the AccumuloRyaQueryEngine used by CreatePCJ.", e);
+ }
+ }
+
+ private static void writeBatch(final FluoClient fluo, final Set<RyaStatement> batch) {
+ checkNotNull(fluo);
+ checkNotNull(batch);
+ new InsertTriples().insert(fluo, batch);
+ }
+
+ private static RyaStatement spToRyaStatement(final StatementPattern sp) {
+ final Value subjVal = sp.getSubjectVar().getValue();
+ final Value predVal = sp.getPredicateVar().getValue();
+ final Value objVal = sp.getObjectVar().getValue();
+
+ RyaURI subjURI = null;
+ RyaURI predURI = null;
+ RyaType objType = null;
+
+ if(subjVal != null) {
+ if(!(subjVal instanceof Resource)) {
+ throw new AssertionError("Subject must be a Resource.");
+ }
+ subjURI = RdfToRyaConversions.convertResource((Resource) subjVal);
+ }
+
+ if (predVal != null) {
+ if(!(predVal instanceof URI)) {
+ throw new AssertionError("Predicate must be a URI.");
+ }
+ predURI = RdfToRyaConversions.convertURI((URI) predVal);
+ }
+
+ if (objVal != null ) {
+ objType = RdfToRyaConversions.convertValue(objVal);
+ }
+
+ return new RyaStatement(subjURI, predURI, objType);
+ }
+
+ private String[] getAuths(final Connector accumulo) {
+ Authorizations auths;
+ try {
+ auths = accumulo.securityOperations().getUserAuthorizations(accumulo.whoami());
+ final List<byte[]> authList = auths.getAuthorizations();
+ final String[] authArray = new String[authList.size()];
+ for(int i = 0; i < authList.size(); i++){
+ authArray[i] = new String(authList.get(i), "UTF-8");
+ }
+ return authArray;
+ } catch (AccumuloException | AccumuloSecurityException | UnsupportedEncodingException e) {
+ throw new RuntimeException("Cannot read authorizations for user: " + accumulo.whoami());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreatePcj.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreatePcj.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreatePcj.java
deleted file mode 100644
index 767d9d2..0000000
--- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreatePcj.java
+++ /dev/null
@@ -1,450 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.indexing.pcj.fluo.api;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static java.util.Objects.requireNonNull;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.client.Transaction;
-import org.apache.log4j.Logger;
-import org.apache.rya.accumulo.AccumuloRdfConfiguration;
-import org.apache.rya.accumulo.query.AccumuloRyaQueryEngine;
-import org.apache.rya.api.domain.RyaStatement;
-import org.apache.rya.api.domain.RyaType;
-import org.apache.rya.api.domain.RyaURI;
-import org.apache.rya.api.persist.RyaDAOException;
-import org.apache.rya.api.persist.query.BatchRyaQuery;
-import org.apache.rya.api.resolver.RdfToRyaConversions;
-import org.apache.rya.indexing.pcj.fluo.app.FluoStringConverter;
-import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
-import org.apache.rya.indexing.pcj.fluo.app.query.FluoQuery;
-import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns;
-import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryMetadataDAO;
-import org.apache.rya.indexing.pcj.fluo.app.query.QueryMetadata;
-import org.apache.rya.indexing.pcj.fluo.app.query.SparqlFluoQueryBuilder;
-import org.apache.rya.indexing.pcj.fluo.app.query.SparqlFluoQueryBuilder.NodeIds;
-import org.apache.rya.indexing.pcj.fluo.app.query.StatementPatternMetadata;
-import org.apache.rya.indexing.pcj.storage.PcjException;
-import org.apache.rya.indexing.pcj.storage.PcjMetadata;
-import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage;
-import org.calrissian.mango.collect.CloseableIterable;
-import org.openrdf.model.Resource;
-import org.openrdf.model.URI;
-import org.openrdf.model.Value;
-import org.openrdf.query.MalformedQueryException;
-import org.openrdf.query.algebra.StatementPattern;
-import org.openrdf.query.parser.ParsedQuery;
-import org.openrdf.query.parser.sparql.SPARQLParser;
-
-import com.google.common.base.Preconditions;
-
-import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
-import edu.umd.cs.findbugs.annotations.NonNull;
-
-/**
- * Sets up a new Pre Computed Join (PCJ) in Fluo from a SPARQL query.
- * <p>
- * This is a two phase process.
- * <ol>
- * <li>Setup metadata about each node of the query using a single Fluo transaction. </li>
- * <li>Scan Rya for binding sets that match each Statement Pattern from the query
- * and use a separate Fluo transaction for each batch that is inserted. This
- * ensure historic triples will be included in the query's results.</li>
- * </ol>
- * After the first step is finished, any new Triples that are added to the Fluo
- * application will be matched against statement patterns, the final results
- * will percolate to the top of the query, and those results will be exported to
- * Rya's query system.
- */
-@DefaultAnnotation(NonNull.class)
-public class CreatePcj {
- private static final Logger log = Logger.getLogger(CreatePcj.class);
-
- /**
- * The default Statement Pattern batch insert size is 1000.
- */
- private static final int DEFAULT_SP_INSERT_BATCH_SIZE = 1000;
-
- /**
- * The maximum number of binding sets that will be inserted into each Statement
- * Pattern's result set per Fluo transaction.
- */
- private final int spInsertBatchSize;
-
- /**
- * Constructs an instance of {@link CreatePcj} that uses
- * {@link #DEFAULT_SP_INSERT_BATCH_SIZE} as the default batch insert size.
- */
- public CreatePcj() {
- this(DEFAULT_SP_INSERT_BATCH_SIZE);
- }
-
- /**
- * Constructs an instance of {@link CreatePcj}.
- *
- * @param spInsertBatchSize - The maximum number of binding sets that will be
- * inserted into each Statement Pattern's result set per Fluo transaction.
- */
- public CreatePcj(final int spInsertBatchSize) {
- checkArgument(spInsertBatchSize > 0, "The SP insert batch size '" + spInsertBatchSize + "' must be greater than 0.");
- this.spInsertBatchSize = spInsertBatchSize;
- }
-
-
- /**
- * Tells the Fluo PCJ Updater application to maintain a new PCJ. This method does not
- * require a pcjId and does not require a PCJ table to have already been created via {@link PrecomputedJoinStorage}.
- * This method only adds the metadata to the Fluo table to incrementally generate query results. Since there
- * is no PCJ table, the incremental results must be exported to some external queuing service such as Kafka.
- * This method currently only supports SPARQL COSNTRUCT queries, as they only export to Kafka by default.
- *
- * @param sparql - SPARQL query whose results will be updated in the Fluo table
- * @param fluo - A connection to the Fluo application that updates the PCJ index. (not null)
- * @return The metadata that was written to the Fluo application for the PCJ.
- * @throws MalformedQueryException The SPARQL query stored for the {@code pcjId} is malformed.
- * @throws PcjException The PCJ Metadata for {@code pcjId} could not be read from {@code pcjStorage}.
- * @throws RuntimeException If SPARQL query is not a CONSTRUCT query.
- */
- public FluoQuery createFluoPcj(final FluoClient fluo, String sparql) throws MalformedQueryException, PcjException {
- requireNonNull(sparql);
- requireNonNull(fluo);
-
- // Keeps track of the IDs that are assigned to each of the query's nodes in Fluo.
- // We use these IDs later when scanning Rya for historic Statement Pattern matches
- // as well as setting up automatic exports.
- final NodeIds nodeIds = new NodeIds();
- final ParsedQuery parsedQuery = new SPARQLParser().parseQuery(sparql, null);
- final FluoQuery fluoQuery = new SparqlFluoQueryBuilder().make(parsedQuery, nodeIds);
- checkArgument(fluoQuery.getConstructQueryMetadata().isPresent(), "Sparql query: " + sparql + " must begin with a construct.");
-
- try (Transaction tx = fluo.newTransaction()) {
- // Write the query's structure to Fluo.
- new FluoQueryMetadataDAO().write(tx, fluoQuery);
- tx.commit();
- }
-
- return fluoQuery;
- }
-
-
-
-
- /**
- * Tells the Fluo PCJ Updater application to maintain a new PCJ. This method
- * creates the FluoQuery (metadata) inside of Fluo so that results can be incrementally generated
- * inside of Fluo. This method assumes that the user will export the results to Kafka or
- * some other external resource. The export id is equivalent to the queryId that is returned,
- * which is in contrast to the other createPcj methods in this class which accept an external pcjId
- * that is used to identify the Accumulo table or Kafka topic for exporting results.
- *
- * @param sparql - sparql query String to be registered with Fluo
- * @param fluo - A connection to the Fluo application that updates the PCJ index. (not null)
- * @return queryId - The id of the root of the query metadata tree in Fluo
- * @throws MalformedQueryException The SPARQL query stored for the {@code pcjId} is malformed.
- * @throws PcjException The PCJ Metadata for {@code pcjId} could not be read from {@code pcjStorage}.
- */
- public String createPcj(String sparql, FluoClient fluo) throws MalformedQueryException {
- Preconditions.checkNotNull(sparql);
- Preconditions.checkNotNull(fluo);
-
- FluoQuery fluoQuery = makeFluoQuery(sparql);
- String queryId = null;
- if(fluoQuery.getQueryMetadata().isPresent()) {
- queryId = fluoQuery.getQueryMetadata().get().getNodeId();
- queryId = queryId.split(IncrementalUpdateConstants.QUERY_PREFIX)[1];
- } else {
- queryId = fluoQuery.getConstructQueryMetadata().get().getNodeId();
- queryId = queryId.split(IncrementalUpdateConstants.CONSTRUCT_PREFIX)[1];
- }
-
- String[] idArray = queryId.split("_");
- String id = idArray[idArray.length - 1];
-
- writeFluoQuery(fluo, fluoQuery, id);
- return id;
- }
-
- /**
- * Tells the Fluo PCJ Updater application to maintain a new PCJ. This method provides
- * no guarantees that a PCJ with the given pcjId exists outside of Fluo. This method merely
- * creates the FluoQuery (metadata) inside of Fluo so that results and be incrementally generated
- * inside of Fluo. This method assumes that the user will export the results to Kafka or
- * some other external resource.
- *
- * @param pcjId - Identifies the PCJ that will be updated by the Fluo app. (not null)
- * @param sparql - sparql query String to be registered with Fluo
- * @param fluo - A connection to the Fluo application that updates the PCJ index. (not null)
- * @return The metadata that was written to the Fluo application for the PCJ.
- * @throws MalformedQueryException The SPARQL query stored for the {@code pcjId} is malformed.
- * @throws PcjException The PCJ Metadata for {@code pcjId} could not be read from {@code pcjStorage}.
- */
- public FluoQuery createPcj(
- final String pcjId,
- final String sparql,
- final FluoClient fluo) throws MalformedQueryException, PcjException {
- requireNonNull(pcjId);
- requireNonNull(sparql);
- requireNonNull(fluo);
-
- FluoQuery fluoQuery = makeFluoQuery(sparql);
- writeFluoQuery(fluo, fluoQuery, pcjId);
-
- return fluoQuery;
- }
-
- private FluoQuery makeFluoQuery(String sparql) throws MalformedQueryException {
-
- // Keeps track of the IDs that are assigned to each of the query's nodes in Fluo.
- // We use these IDs later when scanning Rya for historic Statement Pattern matches
- // as well as setting up automatic exports.
- final NodeIds nodeIds = new NodeIds();
-
- // Parse the query's structure for the metadata that will be written to fluo.
- final ParsedQuery parsedQuery = new SPARQLParser().parseQuery(sparql, null);
- return new SparqlFluoQueryBuilder().make(parsedQuery, nodeIds);
- }
-
- private void writeFluoQuery(FluoClient fluo, FluoQuery fluoQuery, String pcjId) {
- try (Transaction tx = fluo.newTransaction()) {
- // Write the query's structure to Fluo.
- new FluoQueryMetadataDAO().write(tx, fluoQuery);
-
- // The results of the query are eventually exported to an instance
- // of Rya, so store the Rya ID for the PCJ.
- QueryMetadata metadata = fluoQuery.getQueryMetadata().orNull();
- if (metadata != null) {
- String queryId = metadata.getNodeId();
- tx.set(queryId, FluoQueryColumns.RYA_PCJ_ID, pcjId);
- tx.set(pcjId, FluoQueryColumns.PCJ_ID_QUERY_ID, queryId);
- }
-
- // Flush the changes to Fluo.
- tx.commit();
- }
- }
-
-
- /**
- * Tells the Fluo PCJ Updater application to maintain a new PCJ. The method takes in an
- * instance of {@link PrecomputedJoinStorage} to verify that a PCJ with the given pcjId exists.
- *
- * @param pcjId - Identifies the PCJ that will be updated by the Fluo app. (not null)
- * @param pcjStorage - Provides access to the PCJ index. (not null)
- * @param fluo - A connection to the Fluo application that updates the PCJ index. (not null)
- * @return The metadata that was written to the Fluo application for the PCJ.
- * @throws MalformedQueryException The SPARQL query stored for the {@code pcjId} is malformed.
- * @throws PcjException The PCJ Metadata for {@code pcjId} could not be read from {@code pcjStorage}.
- */
- public FluoQuery createPcj(
- final String pcjId,
- final PrecomputedJoinStorage pcjStorage,
- final FluoClient fluo) throws MalformedQueryException, PcjException {
- requireNonNull(pcjId);
- requireNonNull(pcjStorage);
- requireNonNull(fluo);
-
- // Parse the query's structure for the metadata that will be written to fluo.
- final PcjMetadata pcjMetadata = pcjStorage.getPcjMetadata(pcjId);
- final String sparql = pcjMetadata.getSparql();
- return createPcj(pcjId, sparql, fluo);
- }
-
- /**
- * Tells the Fluo PCJ Updater application to maintain a new PCJ.
- * <p>
- * This call scans Rya for Statement Pattern matches and inserts them into
- * the Fluo application. This method does not verify that a PcjTable with the
- * the given pcjId actually exists. It is assumed that results for any query registered
- * using this method will be exported to Kafka or some other external service.
- *
- * @param pcjId - Identifies the PCJ that will be updated by the Fluo app. (not null)
- * @param sparql - sparql query that will registered with Fluo. (not null)
- * @param fluo - A connection to the Fluo application that updates the PCJ index. (not null)
- * @param queryEngine - QueryEngine for a given Rya Instance, (not null)
- * @return The Fluo application's Query ID of the query that was created.
- * @throws MalformedQueryException The SPARQL query stored for the {@code pcjId} is malformed.
- * @throws PcjException The PCJ Metadata for {@code pcjId} could not be read from {@code pcjStorage}.
- * @throws RyaDAOException Historic PCJ results could not be loaded because of a problem with {@code rya}.
- */
- public String withRyaIntegration(
- final String pcjId,
- final String sparql,
- final FluoClient fluo,
- final Connector accumulo,
- final String ryaInstance ) throws MalformedQueryException, PcjException, RyaDAOException {
- requireNonNull(pcjId);
- requireNonNull(sparql);
- requireNonNull(fluo);
- requireNonNull(accumulo);
- requireNonNull(ryaInstance);
-
-
- // Write the SPARQL query's structure to the Fluo Application.
- final FluoQuery fluoQuery = createPcj(pcjId, sparql, fluo);
- //import results already ingested into Rya that match query
- importHistoricResultsIntoFluo(fluo, fluoQuery, accumulo, ryaInstance);
- // return queryId to the caller for later monitoring from the export.
- return fluoQuery.getQueryMetadata().get().getNodeId();
- }
-
-
- /**
- * Tells the Fluo PCJ Updater application to maintain a new PCJ.
- * <p>
- * This call scans Rya for Statement Pattern matches and inserts them into
- * the Fluo application. The Fluo application will then maintain the intermediate
- * results as new triples are inserted and export any new query results to the
- * {@code pcjId} within the provided {@code pcjStorage}. This method requires that a
- * PCJ table already exist for the query corresponding to the pcjId. Results will be exported
- * to this table.
- *
- * @param pcjId - Identifies the PCJ that will be updated by the Fluo app. (not null)
- * @param pcjStorage - Provides access to the PCJ index. (not null)
- * @param fluo - A connection to the Fluo application that updates the PCJ index. (not null)
- * @param queryEngine - QueryEngine for a given Rya Instance, (not null)
- * @return The Fluo application's Query ID of the query that was created.
- * @throws MalformedQueryException The SPARQL query stored for the {@code pcjId} is malformed.
- * @throws PcjException The PCJ Metadata for {@code pcjId} could not be read from {@code pcjStorage}.
- * @throws RyaDAOException Historic PCJ results could not be loaded because of a problem with {@code rya}.
- */
- public String withRyaIntegration(
- final String pcjId,
- final PrecomputedJoinStorage pcjStorage,
- final FluoClient fluo,
- final Connector accumulo,
- final String ryaInstance ) throws MalformedQueryException, PcjException, RyaDAOException {
- requireNonNull(pcjId);
- requireNonNull(pcjStorage);
- requireNonNull(fluo);
- requireNonNull(accumulo);
- requireNonNull(ryaInstance);
-
- // Parse the query's structure for the metadata that will be written to fluo.
- final PcjMetadata pcjMetadata = pcjStorage.getPcjMetadata(pcjId);
- final String sparql = pcjMetadata.getSparql();
-
- return withRyaIntegration(pcjId, sparql, fluo, accumulo, ryaInstance);
- }
-
- private void importHistoricResultsIntoFluo(FluoClient fluo, FluoQuery fluoQuery, Connector accumulo, String ryaInstance)
- throws RyaDAOException {
- // Reuse the same set object while performing batch inserts.
- final Set<RyaStatement> queryBatch = new HashSet<>();
-
- // Iterate through each of the statement patterns and insert their
- // historic matches into Fluo.
- for (final StatementPatternMetadata patternMetadata : fluoQuery.getStatementPatternMetadata()) {
- // Get an iterator over all of the binding sets that match the
- // statement pattern.
- final StatementPattern pattern = FluoStringConverter.toStatementPattern(patternMetadata.getStatementPattern());
- queryBatch.add(spToRyaStatement(pattern));
- }
-
- // Create AccumuloRyaQueryEngine to query for historic results
- final AccumuloRdfConfiguration conf = new AccumuloRdfConfiguration();
- conf.setTablePrefix(ryaInstance);
- conf.setAuths(getAuths(accumulo));
-
- try (final AccumuloRyaQueryEngine queryEngine = new AccumuloRyaQueryEngine(accumulo, conf);
- CloseableIterable<RyaStatement> queryIterable = queryEngine.query(new BatchRyaQuery(queryBatch))) {
- final Set<RyaStatement> triplesBatch = new HashSet<>();
-
- // Insert batches of the binding sets into Fluo.
- for (final RyaStatement ryaStatement : queryIterable) {
- if (triplesBatch.size() == spInsertBatchSize) {
- writeBatch(fluo, triplesBatch);
- triplesBatch.clear();
- }
-
- triplesBatch.add(ryaStatement);
- }
-
- if (!triplesBatch.isEmpty()) {
- writeBatch(fluo, triplesBatch);
- triplesBatch.clear();
- }
- } catch (final IOException e) {
- log.warn("Ignoring IOException thrown while closing the AccumuloRyaQueryEngine used by CreatePCJ.", e);
- }
- }
-
- private static void writeBatch(final FluoClient fluo, final Set<RyaStatement> batch) {
- checkNotNull(fluo);
- checkNotNull(batch);
- new InsertTriples().insert(fluo, batch);
- }
-
- private static RyaStatement spToRyaStatement(final StatementPattern sp) {
- final Value subjVal = sp.getSubjectVar().getValue();
- final Value predVal = sp.getPredicateVar().getValue();
- final Value objVal = sp.getObjectVar().getValue();
-
- RyaURI subjURI = null;
- RyaURI predURI = null;
- RyaType objType = null;
-
- if(subjVal != null) {
- if(!(subjVal instanceof Resource)) {
- throw new AssertionError("Subject must be a Resource.");
- }
- subjURI = RdfToRyaConversions.convertResource((Resource) subjVal);
- }
-
- if (predVal != null) {
- if(!(predVal instanceof URI)) {
- throw new AssertionError("Predicate must be a URI.");
- }
- predURI = RdfToRyaConversions.convertURI((URI) predVal);
- }
-
- if (objVal != null ) {
- objType = RdfToRyaConversions.convertValue(objVal);
- }
-
- return new RyaStatement(subjURI, predURI, objType);
- }
-
- private String[] getAuths(final Connector accumulo) {
- Authorizations auths;
- try {
- auths = accumulo.securityOperations().getUserAuthorizations(accumulo.whoami());
- final List<byte[]> authList = auths.getAuthorizations();
- final String[] authArray = new String[authList.size()];
- for(int i = 0; i < authList.size(); i++){
- authArray[i] = new String(authList.get(i), "UTF-8");
- }
- return authArray;
- } catch (AccumuloException | AccumuloSecurityException | UnsupportedEncodingException e) {
- throw new RuntimeException("Cannot read authorizations for user: " + accumulo.whoami());
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeleteFluoPcj.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeleteFluoPcj.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeleteFluoPcj.java
new file mode 100644
index 0000000..58a52fb
--- /dev/null
+++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeleteFluoPcj.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.fluo.api;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Objects.requireNonNull;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.fluo.api.client.FluoClient;
+import org.apache.fluo.api.client.Transaction;
+import org.apache.fluo.api.client.scanner.CellScanner;
+import org.apache.fluo.api.data.Bytes;
+import org.apache.fluo.api.data.Column;
+import org.apache.fluo.api.data.RowColumnValue;
+import org.apache.fluo.api.data.Span;
+import org.apache.rya.indexing.pcj.fluo.app.NodeType;
+import org.apache.rya.indexing.pcj.fluo.app.query.AggregationMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.ConstructQueryMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.FilterMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns;
+import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryMetadataDAO;
+import org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.PeriodicQueryMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.ProjectionMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.QueryMetadata;
+import org.openrdf.query.BindingSet;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
+
+/**
+ * Deletes a Pre-computed Join (PCJ) from Fluo.
+ * <p>
+ * This is a two phase process.
+ * <ol>
+ * <li>Delete metadata about each node of the query using a single Fluo
+ * transaction. This prevents new {@link BindingSet}s from being created when
+ * new triples are inserted.</li>
+ * <li>Delete BindingSets associated with each node of the query. This is done
+ * in a batch fashion to guard against large delete transactions that don't fit
+ * into memory.</li>
+ * </ol>
+ */
+@DefaultAnnotation(NonNull.class)
+public class DeleteFluoPcj {
+
+ private final FluoQueryMetadataDAO dao = new FluoQueryMetadataDAO();
+ private final int batchSize;
+
+ /**
+ * Constructs an instance of {@link DeleteFluoPcj}.
+ *
+ * @param batchSize - The number of entries that will be deleted at a time. (> 0)
+ */
+ public DeleteFluoPcj(final int batchSize) {
+ checkArgument(batchSize > 0);
+ this.batchSize = batchSize;
+ }
+
+ /**
+ * Deletes all metadata and {@link BindingSet}s associated with a Rya
+ * Precomputed Join Index from the Fluo application that is incrementally
+ * updating it.
+ *
+ * @param client - Connects to the Fluo application that is updating the PCJ
+ * Index. (not null)
+ * @param pcjId - The PCJ ID for the query that will removed from the Fluo
+ * application. (not null)
+ */
+ public void deletePcj(final FluoClient client, final String pcjId) {
+ requireNonNull(client);
+ requireNonNull(pcjId);
+
+ final Transaction tx = client.newTransaction();
+
+ // Delete the query's metadata. This halts input.
+ final List<String> nodeIds = getNodeIds(tx, pcjId);
+ deleteMetadata(tx, nodeIds, pcjId);
+
+ // Delete the binding sets associated with the query's nodes.
+ for (final String nodeId : nodeIds) {
+ deleteData(client, nodeId);
+ }
+ }
+
+ /**
+ * This method retrieves all of the nodeIds that are part of the query with
+ * specified pcjId.
+ *
+ * @param tx - Transaction of a given Fluo table. (not null)
+ * @param pcjId - Id of query. (not null)
+ * @return list of Node IDs associated with the query {@code pcjId}.
+ */
+ private List<String> getNodeIds(Transaction tx, String pcjId) {
+ requireNonNull(tx);
+ requireNonNull(pcjId);
+
+ // Get the ID that tracks the query within the Fluo application.
+ final String queryId = getQueryIdFromPcjId(tx, pcjId);
+
+ // Get the query's children nodes.
+ final List<String> nodeIds = new ArrayList<>();
+ nodeIds.add(queryId);
+ getChildNodeIds(tx, queryId, nodeIds);
+ return nodeIds;
+ }
+
+ /**
+ * Recursively navigate query tree to extract all of the nodeIds.
+ *
+ * @param tx - Transaction of a given Fluo table. (not null)
+ * @param nodeId - Current node in query tree. (not null)
+ * @param nodeIds - The Node IDs extracted from query tree. (not null)
+ */
+ private void getChildNodeIds(final Transaction tx, final String nodeId, final List<String> nodeIds) {
+ requireNonNull(tx);
+ requireNonNull(nodeId);
+ requireNonNull(nodeIds);
+
+ final NodeType type = NodeType.fromNodeId(nodeId).get();
+ switch (type) {
+ case QUERY:
+ final QueryMetadata queryMeta = dao.readQueryMetadata(tx, nodeId);
+ final String queryChild = queryMeta.getChildNodeId();
+ nodeIds.add(queryChild);
+ getChildNodeIds(tx, queryChild, nodeIds);
+ break;
+ case CONSTRUCT:
+ final ConstructQueryMetadata constructMeta = dao.readConstructQueryMetadata(tx, nodeId);
+ final String constructChild = constructMeta.getChildNodeId();
+ nodeIds.add(constructChild);
+ getChildNodeIds(tx, constructChild, nodeIds);
+ break;
+ case JOIN:
+ final JoinMetadata joinMeta = dao.readJoinMetadata(tx, nodeId);
+ final String lchild = joinMeta.getLeftChildNodeId();
+ final String rchild = joinMeta.getRightChildNodeId();
+ nodeIds.add(lchild);
+ nodeIds.add(rchild);
+ getChildNodeIds(tx, lchild, nodeIds);
+ getChildNodeIds(tx, rchild, nodeIds);
+ break;
+ case FILTER:
+ final FilterMetadata filterMeta = dao.readFilterMetadata(tx, nodeId);
+ final String filterChild = filterMeta.getChildNodeId();
+ nodeIds.add(filterChild);
+ getChildNodeIds(tx, filterChild, nodeIds);
+ break;
+ case AGGREGATION:
+ final AggregationMetadata aggMeta = dao.readAggregationMetadata(tx, nodeId);
+ final String aggChild = aggMeta.getChildNodeId();
+ nodeIds.add(aggChild);
+ getChildNodeIds(tx, aggChild, nodeIds);
+ break;
+ case PERIODIC_QUERY:
+ final PeriodicQueryMetadata periodicMeta = dao.readPeriodicQueryMetadata(tx, nodeId);
+ final String periodicChild = periodicMeta.getChildNodeId();
+ nodeIds.add(periodicChild);
+ getChildNodeIds(tx, periodicChild, nodeIds);
+ break;
+ case PROJECTION:
+ final ProjectionMetadata projectionMetadata = dao.readProjectionMetadata(tx, nodeId);
+ final String projectionChild = projectionMetadata.getChildNodeId();
+ nodeIds.add(projectionChild);
+ getChildNodeIds(tx, projectionChild, nodeIds);
+ break;
+ case STATEMENT_PATTERN:
+ break;
+ }
+ }
+
+ /**
+ * Deletes metadata for all nodeIds associated with a given queryId in a
+ * single transaction. Prevents additional BindingSets from being created as
+ * new triples are added.
+ *
+ * @param tx - Transaction of a given Fluo table. (not null)
+ * @param nodeIds - Nodes whose metatdata will be deleted. (not null)
+ * @param pcjId - The PCJ ID of the query whose will be deleted. (not null)
+ */
+ private void deleteMetadata(final Transaction tx, final List<String> nodeIds, final String pcjId) {
+ requireNonNull(tx);
+ requireNonNull(nodeIds);
+ requireNonNull(pcjId);
+
+ try (final Transaction typeTx = tx) {
+ deletePcjIdAndSparqlMetadata(typeTx, pcjId);
+
+ for (final String nodeId : nodeIds) {
+ final NodeType type = NodeType.fromNodeId(nodeId).get();
+ deleteMetadataColumns(typeTx, nodeId, type.getMetaDataColumns());
+ }
+ typeTx.commit();
+ }
+ }
+
+ /**
+ * Deletes all metadata for a Query Node.
+ *
+ * @param tx - Transaction the deletes will be performed with. (not null)
+ * @param nodeId - The Node ID of the query node to delete. (not null)
+ * @param columns - The columns that will be deleted. (not null)
+ */
+ private void deleteMetadataColumns(final Transaction tx, final String nodeId, final List<Column> columns) {
+ requireNonNull(tx);
+ requireNonNull(columns);
+ requireNonNull(nodeId);
+
+ final Bytes row = Bytes.of(nodeId);
+ for (final Column column : columns) {
+ tx.delete(row, column);
+ }
+ }
+
+ /**
+ * Deletes high level query meta for converting from queryId to pcjId and
+ * vice versa, as well as converting from sparql to queryId.
+ *
+ * @param tx - Transaction the deletes will be performed with. (not null)
+ * @param pcjId - The PCJ whose metadata will be deleted. (not null)
+ */
+ private void deletePcjIdAndSparqlMetadata(final Transaction tx, final String pcjId) {
+ requireNonNull(tx);
+ requireNonNull(pcjId);
+
+ final String queryId = getQueryIdFromPcjId(tx, pcjId);
+ final String sparql = getSparqlFromQueryId(tx, queryId);
+ tx.delete(queryId, FluoQueryColumns.RYA_PCJ_ID);
+ tx.delete(sparql, FluoQueryColumns.QUERY_ID);
+ tx.delete(pcjId, FluoQueryColumns.PCJ_ID_QUERY_ID);
+ }
+
+ /**
+ * Deletes all results (BindingSets or Statements) associated with the specified nodeId.
+ *
+ * @param nodeId - nodeId whose {@link BindingSet}s will be deleted. (not null)
+ * @param client - Used to delete the data. (not null)
+ */
+ private void deleteData(final FluoClient client, final String nodeId) {
+ requireNonNull(client);
+ requireNonNull(nodeId);
+
+ final NodeType type = NodeType.fromNodeId(nodeId).get();
+ Transaction tx = client.newTransaction();
+ while (deleteDataBatch(tx, getIterator(tx, nodeId, type.getResultColumn()), type.getResultColumn())) {
+ tx = client.newTransaction();
+ }
+ }
+
+ private CellScanner getIterator(final Transaction tx, final String nodeId, final Column column) {
+ requireNonNull(tx);
+ requireNonNull(nodeId);
+ requireNonNull(column);
+
+ return tx.scanner().fetch(column).over(Span.prefix(nodeId)).build();
+ }
+
+ private boolean deleteDataBatch(final Transaction tx, final CellScanner scanner, final Column column) {
+ requireNonNull(tx);
+ requireNonNull(scanner);
+ requireNonNull(column);
+
+ try (Transaction ntx = tx) {
+ int count = 0;
+ final Iterator<RowColumnValue> iter = scanner.iterator();
+ while (iter.hasNext() && count < batchSize) {
+ final Bytes row = iter.next().getRow();
+ count++;
+ tx.delete(row, column);
+ }
+
+ final boolean hasNext = iter.hasNext();
+ tx.commit();
+ return hasNext;
+ }
+ }
+
+ private String getQueryIdFromPcjId(final Transaction tx, final String pcjId) {
+ requireNonNull(tx);
+ requireNonNull(pcjId);
+
+ final Bytes queryIdBytes = tx.get(Bytes.of(pcjId), FluoQueryColumns.PCJ_ID_QUERY_ID);
+ return queryIdBytes.toString();
+ }
+
+ private String getSparqlFromQueryId(final Transaction tx, final String queryId) {
+ requireNonNull(tx);
+ requireNonNull(queryId);
+
+ final QueryMetadata metadata = dao.readQueryMetadata(tx, queryId);
+ return metadata.getSparql();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeletePcj.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeletePcj.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeletePcj.java
deleted file mode 100644
index 3052c1d..0000000
--- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeletePcj.java
+++ /dev/null
@@ -1,305 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.rya.indexing.pcj.fluo.api;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static java.util.Objects.requireNonNull;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.fluo.api.client.FluoClient;
-import org.apache.fluo.api.client.Transaction;
-import org.apache.fluo.api.client.scanner.CellScanner;
-import org.apache.fluo.api.data.Bytes;
-import org.apache.fluo.api.data.Column;
-import org.apache.fluo.api.data.RowColumnValue;
-import org.apache.fluo.api.data.Span;
-import org.apache.rya.indexing.pcj.fluo.app.NodeType;
-import org.apache.rya.indexing.pcj.fluo.app.query.AggregationMetadata;
-import org.apache.rya.indexing.pcj.fluo.app.query.ConstructQueryMetadata;
-import org.apache.rya.indexing.pcj.fluo.app.query.FilterMetadata;
-import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns;
-import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryMetadataDAO;
-import org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata;
-import org.apache.rya.indexing.pcj.fluo.app.query.PeriodicQueryMetadata;
-import org.apache.rya.indexing.pcj.fluo.app.query.QueryMetadata;
-import org.openrdf.query.BindingSet;
-
-import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
-import edu.umd.cs.findbugs.annotations.NonNull;
-
-/**
- * Deletes a Pre-computed Join (PCJ) from Fluo.
- * <p>
- * This is a two phase process.
- * <ol>
- * <li>Delete metadata about each node of the query using a single Fluo
- * transaction. This prevents new {@link BindingSet}s from being created when
- * new triples are inserted.</li>
- * <li>Delete BindingSets associated with each node of the query. This is done
- * in a batch fashion to guard against large delete transactions that don't fit
- * into memory.</li>
- * </ol>
- */
-@DefaultAnnotation(NonNull.class)
-public class DeletePcj {
-
- private final FluoQueryMetadataDAO dao = new FluoQueryMetadataDAO();
- private final int batchSize;
-
- /**
- * Constructs an instance of {@link DeletePcj}.
- *
- * @param batchSize - The number of entries that will be deleted at a time. (> 0)
- */
- public DeletePcj(final int batchSize) {
- checkArgument(batchSize > 0);
- this.batchSize = batchSize;
- }
-
- /**
- * Deletes all metadata and {@link BindingSet}s associated with a Rya
- * Precomputed Join Index from the Fluo application that is incrementally
- * updating it.
- *
- * @param client - Connects to the Fluo application that is updating the PCJ
- * Index. (not null)
- * @param pcjId - The PCJ ID for the query that will removed from the Fluo
- * application. (not null)
- */
- public void deletePcj(final FluoClient client, final String pcjId) {
- requireNonNull(client);
- requireNonNull(pcjId);
-
- final Transaction tx = client.newTransaction();
-
- // Delete the query's metadata. This halts input.
- final List<String> nodeIds = getNodeIds(tx, pcjId);
- deleteMetadata(tx, nodeIds, pcjId);
-
- // Delete the binding sets associated with the query's nodes.
- for (final String nodeId : nodeIds) {
- deleteData(client, nodeId);
- }
- }
-
- /**
- * This method retrieves all of the nodeIds that are part of the query with
- * specified pcjId.
- *
- * @param tx - Transaction of a given Fluo table. (not null)
- * @param pcjId - Id of query. (not null)
- * @return list of Node IDs associated with the query {@code pcjId}.
- */
- private List<String> getNodeIds(Transaction tx, String pcjId) {
- requireNonNull(tx);
- requireNonNull(pcjId);
-
- // Get the ID that tracks the query within the Fluo application.
- final String queryId = getQueryIdFromPcjId(tx, pcjId);
-
- // Get the query's children nodes.
- final List<String> nodeIds = new ArrayList<>();
- nodeIds.add(queryId);
- getChildNodeIds(tx, queryId, nodeIds);
- return nodeIds;
- }
-
- /**
- * Recursively navigate query tree to extract all of the nodeIds.
- *
- * @param tx - Transaction of a given Fluo table. (not null)
- * @param nodeId - Current node in query tree. (not null)
- * @param nodeIds - The Node IDs extracted from query tree. (not null)
- */
- private void getChildNodeIds(final Transaction tx, final String nodeId, final List<String> nodeIds) {
- requireNonNull(tx);
- requireNonNull(nodeId);
- requireNonNull(nodeIds);
-
- final NodeType type = NodeType.fromNodeId(nodeId).get();
- switch (type) {
- case QUERY:
- final QueryMetadata queryMeta = dao.readQueryMetadata(tx, nodeId);
- final String queryChild = queryMeta.getChildNodeId();
- nodeIds.add(queryChild);
- getChildNodeIds(tx, queryChild, nodeIds);
- break;
- case CONSTRUCT:
- final ConstructQueryMetadata constructMeta = dao.readConstructQueryMetadata(tx, nodeId);
- final String constructChild = constructMeta.getChildNodeId();
- nodeIds.add(constructChild);
- getChildNodeIds(tx, constructChild, nodeIds);
- break;
- case JOIN:
- final JoinMetadata joinMeta = dao.readJoinMetadata(tx, nodeId);
- final String lchild = joinMeta.getLeftChildNodeId();
- final String rchild = joinMeta.getRightChildNodeId();
- nodeIds.add(lchild);
- nodeIds.add(rchild);
- getChildNodeIds(tx, lchild, nodeIds);
- getChildNodeIds(tx, rchild, nodeIds);
- break;
- case FILTER:
- final FilterMetadata filterMeta = dao.readFilterMetadata(tx, nodeId);
- final String filterChild = filterMeta.getChildNodeId();
- nodeIds.add(filterChild);
- getChildNodeIds(tx, filterChild, nodeIds);
- break;
- case AGGREGATION:
- final AggregationMetadata aggMeta = dao.readAggregationMetadata(tx, nodeId);
- final String aggChild = aggMeta.getChildNodeId();
- nodeIds.add(aggChild);
- getChildNodeIds(tx, aggChild, nodeIds);
- break;
- case PERIODIC_QUERY:
- final PeriodicQueryMetadata periodicMeta = dao.readPeriodicQueryMetadata(tx, nodeId);
- final String periodicChild = periodicMeta.getChildNodeId();
- nodeIds.add(periodicChild);
- getChildNodeIds(tx, periodicChild, nodeIds);
- break;
- case STATEMENT_PATTERN:
- break;
- }
- }
-
- /**
- * Deletes metadata for all nodeIds associated with a given queryId in a
- * single transaction. Prevents additional BindingSets from being created as
- * new triples are added.
- *
- * @param tx - Transaction of a given Fluo table. (not null)
- * @param nodeIds - Nodes whose metatdata will be deleted. (not null)
- * @param pcjId - The PCJ ID of the query whose will be deleted. (not null)
- */
- private void deleteMetadata(final Transaction tx, final List<String> nodeIds, final String pcjId) {
- requireNonNull(tx);
- requireNonNull(nodeIds);
- requireNonNull(pcjId);
-
- try (final Transaction typeTx = tx) {
- deletePcjIdAndSparqlMetadata(typeTx, pcjId);
-
- for (final String nodeId : nodeIds) {
- final NodeType type = NodeType.fromNodeId(nodeId).get();
- deleteMetadataColumns(typeTx, nodeId, type.getMetaDataColumns());
- }
- typeTx.commit();
- }
- }
-
- /**
- * Deletes all metadata for a Query Node.
- *
- * @param tx - Transaction the deletes will be performed with. (not null)
- * @param nodeId - The Node ID of the query node to delete. (not null)
- * @param columns - The columns that will be deleted. (not null)
- */
- private void deleteMetadataColumns(final Transaction tx, final String nodeId, final List<Column> columns) {
- requireNonNull(tx);
- requireNonNull(columns);
- requireNonNull(nodeId);
-
- final Bytes row = Bytes.of(nodeId);
- for (final Column column : columns) {
- tx.delete(row, column);
- }
- }
-
- /**
- * Deletes high level query meta for converting from queryId to pcjId and
- * vice versa, as well as converting from sparql to queryId.
- *
- * @param tx - Transaction the deletes will be performed with. (not null)
- * @param pcjId - The PCJ whose metadata will be deleted. (not null)
- */
- private void deletePcjIdAndSparqlMetadata(final Transaction tx, final String pcjId) {
- requireNonNull(tx);
- requireNonNull(pcjId);
-
- final String queryId = getQueryIdFromPcjId(tx, pcjId);
- final String sparql = getSparqlFromQueryId(tx, queryId);
- tx.delete(queryId, FluoQueryColumns.RYA_PCJ_ID);
- tx.delete(sparql, FluoQueryColumns.QUERY_ID);
- tx.delete(pcjId, FluoQueryColumns.PCJ_ID_QUERY_ID);
- }
-
- /**
- * Deletes all results (BindingSets or Statements) associated with the specified nodeId.
- *
- * @param nodeId - nodeId whose {@link BindingSet}s will be deleted. (not null)
- * @param client - Used to delete the data. (not null)
- */
- private void deleteData(final FluoClient client, final String nodeId) {
- requireNonNull(client);
- requireNonNull(nodeId);
-
- final NodeType type = NodeType.fromNodeId(nodeId).get();
- Transaction tx = client.newTransaction();
- while (deleteDataBatch(tx, getIterator(tx, nodeId, type.getResultColumn()), type.getResultColumn())) {
- tx = client.newTransaction();
- }
- }
-
- private CellScanner getIterator(final Transaction tx, final String nodeId, final Column column) {
- requireNonNull(tx);
- requireNonNull(nodeId);
- requireNonNull(column);
-
- return tx.scanner().fetch(column).over(Span.prefix(nodeId)).build();
- }
-
- private boolean deleteDataBatch(final Transaction tx, final CellScanner scanner, final Column column) {
- requireNonNull(tx);
- requireNonNull(scanner);
- requireNonNull(column);
-
- try (Transaction ntx = tx) {
- int count = 0;
- final Iterator<RowColumnValue> iter = scanner.iterator();
- while (iter.hasNext() && count < batchSize) {
- final Bytes row = iter.next().getRow();
- count++;
- tx.delete(row, column);
- }
-
- final boolean hasNext = iter.hasNext();
- tx.commit();
- return hasNext;
- }
- }
-
- private String getQueryIdFromPcjId(final Transaction tx, final String pcjId) {
- requireNonNull(tx);
- requireNonNull(pcjId);
-
- final Bytes queryIdBytes = tx.get(Bytes.of(pcjId), FluoQueryColumns.PCJ_ID_QUERY_ID);
- return queryIdBytes.toString();
- }
-
- private String getSparqlFromQueryId(final Transaction tx, final String queryId) {
- requireNonNull(tx);
- requireNonNull(queryId);
-
- final QueryMetadata metadata = dao.readQueryMetadata(tx, queryId);
- return metadata.getSparql();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ConstructProjection.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ConstructProjection.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ConstructProjection.java
index 6c1aa01..76b62d8 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ConstructProjection.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ConstructProjection.java
@@ -19,12 +19,8 @@ package org.apache.rya.indexing.pcj.fluo.app;
* under the License.
*/
import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.UUID;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.log4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ConstructQueryResultUpdater.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ConstructQueryResultUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ConstructQueryResultUpdater.java
index d8d60b5..6642780 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ConstructQueryResultUpdater.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/ConstructQueryResultUpdater.java
@@ -23,12 +23,13 @@ import org.apache.fluo.api.client.TransactionBase;
import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.Column;
import org.apache.log4j.Logger;
-import org.apache.rya.api.domain.RyaSchema;
import org.apache.rya.api.domain.RyaStatement;
import org.apache.rya.api.domain.RyaSubGraph;
import org.apache.rya.indexing.pcj.fluo.app.export.kafka.RyaSubGraphKafkaSerDe;
import org.apache.rya.indexing.pcj.fluo.app.query.ConstructQueryMetadata;
import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns;
+import org.apache.rya.indexing.pcj.fluo.app.util.RowKeyUtil;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet;
/**
@@ -53,39 +54,26 @@ public class ConstructQueryResultUpdater {
public void updateConstructQueryResults(TransactionBase tx, VisibilityBindingSet bs, ConstructQueryMetadata metadata) {
String nodeId = metadata.getNodeId();
+ VariableOrder varOrder = metadata.getVariableOrder();
Column column = FluoQueryColumns.CONSTRUCT_STATEMENTS;
ConstructGraph graph = metadata.getConstructGraph();
+ String parentId = metadata.getParentNodeId();
- try {
+ // Create the Row Key for the emitted binding set. It does not contain visibilities.
+ final Bytes resultRow = RowKeyUtil.makeRowKey(nodeId, varOrder, bs);
+
+ // If this is a new binding set, then emit it.
+ if(tx.get(resultRow, column) == null || varOrder.getVariableOrders().size() < bs.size()) {
Set<RyaStatement> statements = graph.createGraphFromBindingSet(bs);
- RyaSubGraph subgraph = new RyaSubGraph(metadata.getNodeId(), statements);
- String resultId = nodeId + "_" + getSubGraphId(subgraph);
- tx.set(Bytes.of(resultId), column, Bytes.of(serializer.toBytes(subgraph)));
- } catch (Exception e) {
- log.trace("Unable to serialize RyaStatement generated by ConstructGraph: " + graph + " from BindingSet: " + bs );
- }
- }
-
- /**
- * Generates a simple hash used as an id for the subgraph. Id generated as hash as opposed
- * to UUID to avoid the same subgraph result being stored under multiple UUID.
- * @param subgraph - subgraph that an id is need for
- * @return - hash of subgraph used as an id
- */
- private int getSubGraphId(RyaSubGraph subgraph) {
- int id = 17;
- id = 31*id + subgraph.getId().hashCode();
- for(RyaStatement statement: subgraph.getStatements()) {
- int statementId = 7;
- if(!statement.getSubject().getData().startsWith(RyaSchema.BNODE_NAMESPACE)) {
- statementId = 17*statementId + statement.getSubject().hashCode();
- }
- statementId = 17*statementId + statement.getPredicate().hashCode();
- statementId = 17*statementId + statement.getObject().hashCode();
- id += statementId;
+ RyaSubGraph subgraph = new RyaSubGraph(parentId, statements);
+ final Bytes nodeValueBytes = Bytes.of(serializer.toBytes(subgraph));
+
+ log.trace(
+ "Transaction ID: " + tx.getStartTimestamp() + "\n" +
+ "New Binding Set: " + subgraph + "\n");
+
+ tx.set(resultRow, column, nodeValueBytes);
}
- return Math.abs(id);
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FilterResultUpdater.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FilterResultUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FilterResultUpdater.java
index 1c99051..7cfa216 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FilterResultUpdater.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FilterResultUpdater.java
@@ -25,7 +25,6 @@ import org.apache.fluo.api.data.Bytes;
import org.apache.log4j.Logger;
import org.apache.rya.indexing.pcj.fluo.app.query.FilterMetadata;
import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns;
-import org.apache.rya.indexing.pcj.fluo.app.util.BindingSetUtil;
import org.apache.rya.indexing.pcj.fluo.app.util.FilterSerializer;
import org.apache.rya.indexing.pcj.fluo.app.util.RowKeyUtil;
import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
@@ -46,8 +45,6 @@ import org.openrdf.query.algebra.evaluation.ValueExprEvaluationException;
import org.openrdf.query.algebra.evaluation.impl.EvaluationStrategyImpl;
import org.openrdf.query.algebra.evaluation.util.QueryEvaluationUtil;
-import com.google.common.base.Optional;
-
import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
import edu.umd.cs.findbugs.annotations.NonNull;
import info.aduna.iteration.CloseableIteration;
@@ -114,24 +111,16 @@ public class FilterResultUpdater {
// Evaluate whether the child BindingSet satisfies the filter's condition.
final ValueExpr condition = filter.getCondition();
if (isTrue(condition, childBindingSet)) {
- // Create the Filter's binding set from the child's.
- final VariableOrder filterVarOrder = filterMetadata.getVariableOrder();
- final BindingSet filterBindingSet = BindingSetUtil.keepBindings(filterVarOrder, childBindingSet);
// Create the Row Key for the emitted binding set. It does not contain visibilities.
- final Bytes resultRow = RowKeyUtil.makeRowKey(filterMetadata.getNodeId(), filterVarOrder, filterBindingSet);
-
- // If this is a new binding set, then emit it.
- if(tx.get(resultRow, FluoQueryColumns.FILTER_BINDING_SET) == null) {
- final VisibilityBindingSet visBindingSet = new VisibilityBindingSet(filterBindingSet, childBindingSet.getVisibility());
- final Bytes nodeValueBytes = BS_SERDE.serialize(visBindingSet);
+ final VariableOrder filterVarOrder = filterMetadata.getVariableOrder();
+ final Bytes resultRow = RowKeyUtil.makeRowKey(filterMetadata.getNodeId(), filterVarOrder, childBindingSet);
- log.trace(
- "Transaction ID: " + tx.getStartTimestamp() + "\n" +
- "New Binding Set: " + visBindingSet + "\n");
+ // Serialize and emit BindingSet
+ final Bytes nodeValueBytes = BS_SERDE.serialize(childBindingSet);
+ log.trace("Transaction ID: " + tx.getStartTimestamp() + "\n" + "New Binding Set: " + childBindingSet + "\n");
- tx.set(resultRow, FluoQueryColumns.FILTER_BINDING_SET, nodeValueBytes);
- }
+ tx.set(resultRow, FluoQueryColumns.FILTER_BINDING_SET, nodeValueBytes);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FluoStringConverter.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FluoStringConverter.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FluoStringConverter.java
index 05a8d1c..43a36de 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FluoStringConverter.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FluoStringConverter.java
@@ -23,8 +23,6 @@ import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.DE
import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.TYPE_DELIM;
import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.URI_TYPE;
-import java.util.UUID;
-
import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
import edu.umd.cs.findbugs.annotations.NonNull;
[3/5] incubator-rya git commit: RYA-282-Nested-Query. Closes #192.
Posted by ca...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/ProjectionMetadata.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/ProjectionMetadata.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/ProjectionMetadata.java
new file mode 100644
index 0000000..5a337c7
--- /dev/null
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/ProjectionMetadata.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.fluo.app.query;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+
+import com.google.common.base.Objects;
+
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
+import edu.umd.cs.findbugs.annotations.Nullable;
+import net.jcip.annotations.Immutable;
+
+/**
+ * Metadata that is specific to a Projection.
+ */
+@Immutable
+@DefaultAnnotation(NonNull.class)
+public class ProjectionMetadata extends CommonNodeMetadata {
+
+ private final String childNodeId;
+ private final String parentNodeId;
+ private final VariableOrder projectedVars;
+
+ /**
+ * Constructs an instance of {@link ProjectionMetadata}.
+ *
+ * @param nodeId - The ID the Fluo app uses to reference this node. (not null)
+ * @param varOrder - The order in which binding values are written in the row to identify this result. (not null)
+ * @param childNodeId - The node whose results are projected to the query's SELECT variables. (not null)
+ * @param parentNodeId - The parent node of this projection (not null)
+ * @param projectedVars - The variables that the results are projected onto (not null)
+ */
+ public ProjectionMetadata(
+ final String nodeId,
+ final VariableOrder varOrder,
+ final String childNodeId,
+ final String parentNodeId,
+ final VariableOrder projectedVars) {
+ super(nodeId, varOrder);
+ this.childNodeId = checkNotNull(childNodeId);
+ this.parentNodeId = checkNotNull(parentNodeId);
+ this.projectedVars = checkNotNull(projectedVars);
+ }
+
+ /**
+ * @return The node whose results are projected to the query's SELECT variables.l
+ */
+ public String getChildNodeId() {
+ return childNodeId;
+ }
+
+ /**
+ * @return The parent node of this projection node
+ */
+ public String getParentNodeId() {
+ return parentNodeId;
+ }
+
+ /**
+ * @return The variables that results are projected onto
+ */
+ public VariableOrder getProjectedVars() {
+ return projectedVars;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(
+ super.getNodeId(),
+ super.getVariableOrder(),
+ projectedVars,
+ childNodeId,
+ parentNodeId);
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if(this == o) {
+ return true;
+ }
+
+ if(o instanceof ProjectionMetadata) {
+ if(super.equals(o)) {
+ final ProjectionMetadata projectionMetadata = (ProjectionMetadata)o;
+ return new EqualsBuilder()
+ .append(childNodeId, projectionMetadata.childNodeId)
+ .append(parentNodeId, projectionMetadata.parentNodeId)
+ .append(projectedVars, projectionMetadata.projectedVars)
+ .isEquals();
+ }
+ return false;
+ }
+
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder()
+ .append("ProjectionMetadata {\n")
+ .append(" Node ID: " + super.getNodeId() + "\n")
+ .append(" Projection Variables: " + projectedVars + "\n")
+ .append(" Variable Order: " + super.getVariableOrder() + "\n")
+ .append(" Child Node ID: " + childNodeId + "\n")
+ .append(" Parent Node ID: " + parentNodeId + "\n")
+ .append("}")
+ .toString();
+ }
+
+ /**
+ * Creates a new {@link Builder} for this class.
+ *
+ * @param nodeId - The ID the Fluo app uses to reference this node. (not null)
+ * @return A new {@link Builder} for this class.
+ */
+ public static Builder builder(final String nodeId) {
+ return new Builder(nodeId);
+ }
+
+ /**
+ * Builds instances of {@link ProjectionMetadata}.
+ */
+ @DefaultAnnotation(NonNull.class)
+ public static final class Builder implements CommonNodeMetadata.Builder {
+
+ private String nodeId;
+ private VariableOrder varOrder;
+ private String childNodeId;
+ private String parentNodeId;
+ private VariableOrder projectedVars;
+
+ /**
+ * Constructs an instance of {@link Builder}.
+ *
+ * @param nodeId - The ID the Fluo app uses to reference this node. (not null)
+ */
+ public Builder(final String nodeId) {
+ this.nodeId = checkNotNull(nodeId);
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ /**
+ * Set the variable order of binding sets that are emitted by this node.
+ *
+ * @param varOrder - The order in which result values are written to the row to identify this result
+ * @return This builder so that method invocations may be chained.
+ */
+ public Builder setVarOrder(@Nullable final VariableOrder varOrder) {
+ this.varOrder = varOrder;
+ return this;
+ }
+
+ /**
+ * @return the variable order of binding sets that are emitted by this node
+ */
+ public VariableOrder getVariableOrder() {
+ return varOrder;
+ }
+
+ /**
+ * Set the node whose results are projected to the query's SELECT variables.
+ *
+ * @param childNodeId - The node whose results are projected to the query's SELECT variables.
+ * @return This builder so that method invocations may be chained.
+ */
+ public Builder setChildNodeId(@Nullable final String childNodeId) {
+ this.childNodeId = childNodeId;
+ return this;
+ }
+
+ public String getChildNodeId() {
+ return childNodeId;
+ }
+
+ /**
+ * Set the the parent node of this projection node.
+ *
+ * @param parentNodeId - The parent node of this projection node
+ * @return This builder so that method invocations may be chained.
+ */
+ public Builder setParentNodeId(@Nullable final String parentNodeId) {
+ this.parentNodeId = parentNodeId;
+ return this;
+ }
+
+ public String getParentNodeId() {
+ return parentNodeId;
+ }
+
+ /**
+ * @param varOrder - Variables that results are projected onto
+ * @return This builder so that method invocations may be chained.
+ */
+ public Builder setProjectedVars(VariableOrder projectedVars) {
+ this.projectedVars = projectedVars;
+ return this;
+ }
+
+ /**
+ * @return The variables that results are projected onto
+ */
+ public VariableOrder getProjectionVars() {
+ return projectedVars;
+ }
+
+ /**
+ * @return An instance of {@link ProjectionMetadata} built using this builder's values.
+ */
+ public ProjectionMetadata build() {
+ return new ProjectionMetadata(nodeId, varOrder, childNodeId, parentNodeId, projectedVars);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryBuilderVisitorBase.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryBuilderVisitorBase.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryBuilderVisitorBase.java
new file mode 100644
index 0000000..b45c56c
--- /dev/null
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryBuilderVisitorBase.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.fluo.app.query;
+
+import org.apache.rya.indexing.pcj.fluo.app.NodeType;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+
+/**
+ * Base visitor class for navigating a {@link FluoQuery.Builder}.
+ * The visit methods in this class provide the basic functionality
+ * for navigating between the Builders that make u the FluoQuery.Builder.
+ *
+ */
+public abstract class QueryBuilderVisitorBase {
+
+ private FluoQuery.Builder fluoBuilder;
+
+ public QueryBuilderVisitorBase(FluoQuery.Builder fluoBuilder) {
+ this.fluoBuilder = Preconditions.checkNotNull(fluoBuilder);
+ }
+
+ public void visit() {
+ this.visit(fluoBuilder.getQueryBuilder());
+ }
+
+ /**
+ * Visits the {@link FluoQuery.Builder} starting at the Metadata bulder node with the given id
+ * @param nodeId - id of the node this visitor will start at
+ */
+ public void visit(String nodeId) {
+ visitNode(nodeId);
+ }
+
+ public void visit(QueryMetadata.Builder queryBuilder) {
+ visitNode(queryBuilder.getChildNodeId());
+ }
+
+ public void visit(ConstructQueryMetadata.Builder constructBuilder) {
+ visitNode(constructBuilder.getChildNodeId());
+ }
+
+ public void visit(ProjectionMetadata.Builder projectionBuilder) {
+ visitNode(projectionBuilder.getChildNodeId());
+ }
+
+ public void visit(PeriodicQueryMetadata.Builder periodicBuilder) {
+ visitNode(periodicBuilder.getChildNodeId());
+ }
+
+ public void visit(FilterMetadata.Builder filterBuilder) {
+ visitNode(filterBuilder.getChildNodeId());
+ }
+
+ public void visit(JoinMetadata.Builder joinBuilder) {
+ visitNode(joinBuilder.getLeftChildNodeId());
+ visitNode(joinBuilder.getRightChildNodeId());
+ }
+
+ public void visit(AggregationMetadata.Builder aggregationBuilder) {
+ visitNode(aggregationBuilder.getChildNodeId());
+ }
+
+ public void visit(StatementPatternMetadata.Builder statementPatternBuilder) {}
+
+ public void visitNode(String nodeId) {
+ Optional<NodeType> type = NodeType.fromNodeId(nodeId);
+ try {
+ switch(type.get()) {
+ case AGGREGATION:
+ visit(fluoBuilder.getAggregateBuilder(nodeId).get());
+ break;
+ case CONSTRUCT:
+ visit(fluoBuilder.getConstructQueryBuilder(nodeId).get());
+ break;
+ case FILTER:
+ visit(fluoBuilder.getFilterBuilder(nodeId).get());
+ break;
+ case JOIN:
+ visit(fluoBuilder.getJoinBuilder(nodeId).get());
+ break;
+ case PERIODIC_QUERY:
+ visit(fluoBuilder.getPeriodicQueryBuilder(nodeId).get());
+ break;
+ case PROJECTION:
+ visit(fluoBuilder.getProjectionBuilder(nodeId).get());
+ break;
+ case QUERY:
+ visit(fluoBuilder.getQueryBuilder(nodeId).get());
+ break;
+ case STATEMENT_PATTERN:
+ visit(fluoBuilder.getStatementPatternBuilder(nodeId).get());
+ break;
+ default:
+ throw new RuntimeException();
+ }
+ } catch(Exception e) {
+ throw new IllegalArgumentException("Invalid Fluo Query.");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryMetadata.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryMetadata.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryMetadata.java
index d017724..fe130fb 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryMetadata.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryMetadata.java
@@ -20,18 +20,24 @@ package org.apache.rya.indexing.pcj.fluo.app.query;
import static com.google.common.base.Preconditions.checkNotNull;
-import edu.umd.cs.findbugs.annotations.Nullable;
-import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
-import edu.umd.cs.findbugs.annotations.NonNull;
-import net.jcip.annotations.Immutable;
+import java.util.Set;
import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.ExportStrategy;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.QueryType;
import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
import com.google.common.base.Objects;
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import edu.umd.cs.findbugs.annotations.NonNull;
+import edu.umd.cs.findbugs.annotations.Nullable;
+import net.jcip.annotations.Immutable;
+
/**
- * Metadata that is specific to a Projection.
+ * Metadata for a query registered with Fluo. This metadata is for the topmost node
+ * in the {@link FluoQuery}, and it includes information about how to export results
+ * for the query.
*/
@Immutable
@DefaultAnnotation(NonNull.class)
@@ -39,6 +45,9 @@ public class QueryMetadata extends CommonNodeMetadata {
private final String sparql;
private final String childNodeId;
+ private final Set<ExportStrategy> exportStrategy;
+ private final QueryType queryType;
+ private final String exportId;
/**
* Constructs an instance of {@link QueryMetadata}.
@@ -47,15 +56,29 @@ public class QueryMetadata extends CommonNodeMetadata {
* @param varOrder - The variable order of binding sets that are emitted by this node. (not null)
* @param sparql - The SPARQL query whose results are being updated by the Fluo app. (not null)
* @param childNodeId - The node whose results are projected to the query's SELECT variables. (not null)
+ * @param exportStrategy - Set of export strategies used for emiting results from Rya-Fluo app
*/
public QueryMetadata(
final String nodeId,
final VariableOrder varOrder,
final String sparql,
- final String childNodeId) {
+ final String childNodeId,
+ final Set<ExportStrategy> exportStrategy,
+ final QueryType queryType) {
super(nodeId, varOrder);
this.sparql = checkNotNull(sparql);
this.childNodeId = checkNotNull(childNodeId);
+ this.exportStrategy = checkNotNull(exportStrategy);
+ this.queryType = checkNotNull(queryType);
+ String[] idSplit = nodeId.split("_");
+ if(idSplit.length != 2) {
+ throw new IllegalArgumentException("Invalid Query Node Id");
+ }
+ this.exportId = idSplit[1];
+ }
+
+ public String getExportId() {
+ return exportId;
}
/**
@@ -71,14 +94,30 @@ public class QueryMetadata extends CommonNodeMetadata {
public String getChildNodeId() {
return childNodeId;
}
-
+
+ /**
+ * @return strategies used for exporting results from Rya-Fluo Application
+ */
+ public Set<ExportStrategy> getExportStrategies() {
+ return exportStrategy;
+ }
+
+ /**
+ * @return the {@link QueryType} of this query
+ */
+ public QueryType getQueryType() {
+ return queryType;
+ }
+
@Override
public int hashCode() {
return Objects.hashCode(
super.getNodeId(),
super.getVariableOrder(),
sparql,
- childNodeId);
+ childNodeId,
+ exportStrategy,
+ queryType);
}
@Override
@@ -93,6 +132,8 @@ public class QueryMetadata extends CommonNodeMetadata {
return new EqualsBuilder()
.append(sparql, queryMetadata.sparql)
.append(childNodeId, queryMetadata.childNodeId)
+ .append(exportStrategy, queryMetadata.exportStrategy)
+ .append(queryType, queryMetadata.queryType)
.isEquals();
}
return false;
@@ -109,6 +150,8 @@ public class QueryMetadata extends CommonNodeMetadata {
.append(" Variable Order: " + super.getVariableOrder() + "\n")
.append(" Child Node ID: " + childNodeId + "\n")
.append(" SPARQL: " + sparql + "\n")
+ .append(" Query Type: " + queryType + "\n")
+ .append(" Export Strategies: " + exportStrategy + "\n")
.append("}")
.toString();
}
@@ -127,12 +170,14 @@ public class QueryMetadata extends CommonNodeMetadata {
* Builds instances of {@link QueryMetadata}.
*/
@DefaultAnnotation(NonNull.class)
- public static final class Builder {
+ public static final class Builder implements CommonNodeMetadata.Builder {
private String nodeId;
private VariableOrder varOrder;
private String sparql;
private String childNodeId;
+ private Set<ExportStrategy> exportStrategies;
+ private QueryType queryType;
/**
* Constructs an instance of {@link Builder}.
@@ -154,7 +199,7 @@ public class QueryMetadata extends CommonNodeMetadata {
* @param varOrder - The variable order of binding sets that are emitted by this node.
* @return This builder so that method invocations may be chained.
*/
- public Builder setVariableOrder(@Nullable final VariableOrder varOrder) {
+ public Builder setVarOrder(@Nullable final VariableOrder varOrder) {
this.varOrder = varOrder;
return this;
}
@@ -188,6 +233,37 @@ public class QueryMetadata extends CommonNodeMetadata {
return this;
}
+ /**
+ * Sets export strategies used for emitting results form Rya Fluo app
+ * @param export - Set of export strategies
+ * @return This builder so that method invocations may be chained
+ */
+ public Builder setExportStrategies(Set<ExportStrategy> export) {
+ this.exportStrategies = export;
+ return this;
+ }
+
+ /**
+ * Set query type for the given query
+ * @param queryType - {@link QueryType} of the given query
+ * @return This builder so that method invocations may be chained
+ */
+ public Builder setQueryType(QueryType queryType) {
+ this.queryType = queryType;
+ return this;
+ }
+
+ /**
+ * @return QueryType for the given query
+ */
+ public QueryType getQueryType() {
+ return queryType;
+ }
+
+
+ /**
+ * @return id of the child node of this node
+ */
public String getChildNodeId() {
return childNodeId;
}
@@ -196,7 +272,7 @@ public class QueryMetadata extends CommonNodeMetadata {
* @return An instance of {@link QueryMetadata} build using this builder's values.
*/
public QueryMetadata build() {
- return new QueryMetadata(nodeId, varOrder, sparql, childNodeId);
+ return new QueryMetadata(nodeId, varOrder, sparql, childNodeId, exportStrategies, queryType);
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryMetadataVisitorBase.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryMetadataVisitorBase.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryMetadataVisitorBase.java
new file mode 100644
index 0000000..ce9b02c
--- /dev/null
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryMetadataVisitorBase.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.fluo.app.query;
+
+import org.apache.rya.indexing.pcj.fluo.app.NodeType;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+
+public abstract class QueryMetadataVisitorBase {
+
+ private FluoQuery fluoQuery;
+
+ public QueryMetadataVisitorBase(FluoQuery fluoQuery) {
+ this.fluoQuery = Preconditions.checkNotNull(fluoQuery);
+ }
+
+ public void visit() {
+ visit(fluoQuery.getQueryMetadata());
+ }
+
+ /**
+ * Visits the {@link FluoQuery} starting at the Metadata node with the given id
+ * @param nodeId - id of the node this visitor will start at
+ */
+ public void visit(String nodeId) {
+ visitNode(nodeId);
+ }
+
+ public void visit(QueryMetadata queryMetadata) {
+ visitNode(queryMetadata.getChildNodeId());
+ }
+
+ public void visit(ConstructQueryMetadata constructMetadata) {
+ visitNode(constructMetadata.getChildNodeId());
+ }
+
+ public void visit(ProjectionMetadata projectionMetadata) {
+ visitNode(projectionMetadata.getChildNodeId());
+ }
+
+ public void visit(PeriodicQueryMetadata periodicMetadata) {
+ visitNode(periodicMetadata.getChildNodeId());
+ }
+
+ public void visit(FilterMetadata filterMetadata) {
+ visitNode(filterMetadata.getChildNodeId());
+ }
+
+ public void visit(JoinMetadata joinMetadata) {
+ visitNode(joinMetadata.getLeftChildNodeId());
+ visitNode(joinMetadata.getRightChildNodeId());
+ }
+
+ public void visit(AggregationMetadata aggregationMetadata) {
+ visitNode(aggregationMetadata.getChildNodeId());
+ }
+
+ public void visit(StatementPatternMetadata statementPatternMetadata) {}
+
+ public void visitNode(String nodeId) {
+ Optional<NodeType> type = NodeType.fromNodeId(nodeId);
+ try {
+ switch(type.get()) {
+ case AGGREGATION:
+ visit(fluoQuery.getAggregationMetadata(nodeId).get());
+ break;
+ case CONSTRUCT:
+ visit(fluoQuery.getConstructQueryMetadata(nodeId).get());
+ break;
+ case FILTER:
+ visit(fluoQuery.getFilterMetadata(nodeId).get());
+ break;
+ case JOIN:
+ visit(fluoQuery.getJoinMetadata(nodeId).get());
+ break;
+ case PERIODIC_QUERY:
+ visit(fluoQuery.getPeriodicQueryMetadata(nodeId).get());
+ break;
+ case PROJECTION:
+ visit(fluoQuery.getProjectionMetadata(nodeId).get());
+ break;
+ case QUERY:
+ visit(fluoQuery.getQueryMetadata(nodeId).get());
+ break;
+ case STATEMENT_PATTERN:
+ visit(fluoQuery.getStatementPatternMetadata(nodeId).get());
+ break;
+ default:
+ throw new RuntimeException();
+ }
+ } catch(Exception e) {
+ throw new IllegalArgumentException("Invalid Fluo Query.");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/SparqlFluoQueryBuilder.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/SparqlFluoQueryBuilder.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/SparqlFluoQueryBuilder.java
index 8e348f2..6c03be1 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/SparqlFluoQueryBuilder.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/SparqlFluoQueryBuilder.java
@@ -25,10 +25,11 @@ import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.CO
import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.FILTER_PREFIX;
import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.JOIN_PREFIX;
import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.PERIODIC_QUERY_PREFIX;
-import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.QUERY_PREFIX;
+import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.PROJECTION_PREFIX;
import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.SP_PREFIX;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -42,16 +43,22 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.rya.indexing.pcj.fluo.app.ConstructGraph;
import org.apache.rya.indexing.pcj.fluo.app.ConstructProjection;
import org.apache.rya.indexing.pcj.fluo.app.FluoStringConverter;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.ExportStrategy;
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.QueryType;
import org.apache.rya.indexing.pcj.fluo.app.NodeType;
import org.apache.rya.indexing.pcj.fluo.app.query.AggregationMetadata.AggregationElement;
import org.apache.rya.indexing.pcj.fluo.app.query.AggregationMetadata.AggregationType;
import org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata.JoinType;
import org.apache.rya.indexing.pcj.fluo.app.util.FilterSerializer;
import org.apache.rya.indexing.pcj.fluo.app.util.FilterSerializer.FilterParseException;
+import org.apache.rya.indexing.pcj.fluo.app.util.FluoQueryUtils;
import org.apache.rya.indexing.pcj.fluo.app.util.PeriodicQueryUtil;
+import org.apache.rya.indexing.pcj.fluo.app.util.VariableOrderUpdateVisitor.UpdateAction;
import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
import org.openrdf.model.Value;
import org.openrdf.model.impl.BNodeImpl;
+import org.openrdf.query.MalformedQueryException;
import org.openrdf.query.algebra.AggregateOperator;
import org.openrdf.query.algebra.BNodeGenerator;
import org.openrdf.query.algebra.Extension;
@@ -75,6 +82,7 @@ import org.openrdf.query.algebra.ValueExpr;
import org.openrdf.query.algebra.Var;
import org.openrdf.query.algebra.helpers.QueryModelVisitorBase;
import org.openrdf.query.parser.ParsedQuery;
+import org.openrdf.query.parser.sparql.SPARQLParser;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -90,32 +98,86 @@ import net.jcip.annotations.Immutable;
*/
public class SparqlFluoQueryBuilder {
+ private String sparql;
+ private TupleExpr te;
+ private String queryId;
+ private NodeIds nodeIds;
+ //Default behavior is to export to Kafka - subject to change when user can
+ //specify their own export strategy
+ private Set<ExportStrategy> exportStrategies = new HashSet<>(Arrays.asList(ExportStrategy.Kafka));
+
+ public SparqlFluoQueryBuilder setSparql(String sparql) {
+ this.sparql = Preconditions.checkNotNull(sparql);
+ return this;
+ }
+
+ public SparqlFluoQueryBuilder setTupleExpr(TupleExpr te) {
+ this.te = Preconditions.checkNotNull(te);
+ return this;
+ }
+
/**
- * Creates the {@link FluoQuery} metadata that is required by the Fluo
- * application to process a SPARQL query.
- *
- * @param parsedQuery - The query metadata will be derived from. (not null)
- * @param nodeIds - The NodeIds object is passed in so that other parts
- * of the application may look up which ID is associated with each
- * node of the query.
- * @return A {@link FluoQuery} object loaded with metadata built from the
- * {@link ParsedQuery}.
+ * Sets the FluoQuery id as generated by {@link NodeType#generateNewFluoIdForType(NodeType)} or
+ * {@link NodeType#generateNewIdForType(NodeType, String)}, where NodeType is of type Query.
+ * @param queryId for the {@link FluoQuery}
+ * @return SparqlFluoQueryBuilder for chaining method calls
*/
- public FluoQuery make(final ParsedQuery parsedQuery, final NodeIds nodeIds) {
- checkNotNull(parsedQuery);
-
- final String sparql = parsedQuery.getSourceString();
- final FluoQuery.Builder fluoQueryBuilder = FluoQuery.builder();
-
- final NewQueryVisitor visitor = new NewQueryVisitor(sparql, fluoQueryBuilder, nodeIds);
- TupleExpr te = parsedQuery.getTupleExpr();
+ public SparqlFluoQueryBuilder setFluoQueryId(String queryId) {
+ this.queryId = Preconditions.checkNotNull(queryId);
+ return this;
+ }
+
+ public SparqlFluoQueryBuilder setNodeIds(NodeIds nodeIds) {
+ this.nodeIds = Preconditions.checkNotNull(nodeIds);
+ return this;
+ }
+
+ public SparqlFluoQueryBuilder setExportStrategies(Set<ExportStrategy> exportStrategies) {
+ this.exportStrategies = exportStrategies;
+ return this;
+ }
+
+ public FluoQuery build() {
+ Preconditions.checkNotNull(sparql);
+ Preconditions.checkNotNull(queryId);
+ Preconditions.checkNotNull(exportStrategies);
+
+ if(nodeIds == null) {
+ nodeIds = new NodeIds();
+ }
+
+ if(te == null) {
+ SPARQLParser parser = new SPARQLParser();
+ ParsedQuery pq;
+ try {
+ pq = parser.parseQuery(sparql, null);
+ } catch (MalformedQueryException e) {
+ throw new RuntimeException(e);
+ }
+ te = pq.getTupleExpr();
+ }
+
PeriodicQueryUtil.placePeriodicQueryNode(te);
+ String childNodeId = nodeIds.getOrMakeId(te);
+
+ final FluoQuery.Builder fluoQueryBuilder = FluoQuery.builder();
+ QueryMetadata.Builder queryBuilder = QueryMetadata.builder(queryId);
+ //sets {@link QueryType} and VariableOrder
+ setVarOrderAndQueryType(queryBuilder, te);
+ queryBuilder.setSparql(sparql);
+ queryBuilder.setChildNodeId(childNodeId);
+ queryBuilder.setExportStrategies(exportStrategies);
+ fluoQueryBuilder.setQueryMetadata(queryBuilder);
+
+ setChildMetadata(fluoQueryBuilder, childNodeId, queryBuilder.getVariableOrder(), queryId);
+
+ final NewQueryVisitor visitor = new NewQueryVisitor(fluoQueryBuilder, nodeIds);
te.visit( visitor );
-
+
final FluoQuery fluoQuery = fluoQueryBuilder.build();
return fluoQuery;
}
-
+
/**
* A data structure that creates and keeps track of Node IDs for the nodes
* of a {@link ParsedQuery}. This structure should only be used while creating
@@ -187,7 +249,7 @@ public class SparqlFluoQueryBuilder {
} else if (node instanceof Join || node instanceof LeftJoin) {
prefix = JOIN_PREFIX;
} else if (node instanceof Projection) {
- prefix = QUERY_PREFIX;
+ prefix = PROJECTION_PREFIX;
} else if(node instanceof Extension) {
prefix = AGGREGATION_PREFIX;
} else if (node instanceof Reduced) {
@@ -214,7 +276,6 @@ public class SparqlFluoQueryBuilder {
private final NodeIds nodeIds;
private final FluoQuery.Builder fluoQueryBuilder;
- private final String sparql;
/**
* Constructs an instance of {@link NewQueryVisitor}.
@@ -227,8 +288,7 @@ public class SparqlFluoQueryBuilder {
* of the application may look up which ID is associated with each
* node of the query.
*/
- public NewQueryVisitor(final String sparql, final FluoQuery.Builder fluoQueryBuilder, final NodeIds nodeIds) {
- this.sparql = checkNotNull(sparql);
+ public NewQueryVisitor(final FluoQuery.Builder fluoQueryBuilder, final NodeIds nodeIds) {
this.fluoQueryBuilder = checkNotNull(fluoQueryBuilder);
this.nodeIds = checkNotNull(nodeIds);
}
@@ -256,6 +316,7 @@ public class SparqlFluoQueryBuilder {
} else {
groupByVariableOrder = new VariableOrder();
}
+
// The aggregations that need to be performed are the Group Elements.
final List<AggregationElement> aggregations = new ArrayList<>();
@@ -289,15 +350,21 @@ public class SparqlFluoQueryBuilder {
aggregationBuilder.setChildNodeId(childNodeId);
aggregationBuilder.setGroupByVariableOrder(groupByVariableOrder);
+
+ Set<String> aggregationVars = getVarsToDelete(groupByVariableOrder.getVariableOrders(), aggregationBuilder.getVariableOrder().getVariableOrders());
+ FluoQueryUtils.updateVarOrders(fluoQueryBuilder, UpdateAction.DeleteVariable, Lists.newArrayList(aggregationVars), aggregationId);
+
for(final AggregationElement aggregation : aggregations) {
aggregationBuilder.addAggregation(aggregation);
}
+
+
// Update the child node's metadata.
final Set<String> childVars = getVars(child);
final VariableOrder childVarOrder = new VariableOrder(childVars);
- setChildMetadata(childNodeId, childVarOrder, aggregationId);
+ setChildMetadata(fluoQueryBuilder, childNodeId, childVarOrder, aggregationId);
}
// Walk to the next node.
@@ -369,11 +436,11 @@ public class SparqlFluoQueryBuilder {
// Create or update the left child's variable order and parent node id.
final VariableOrder leftVarOrder = varOrders.getLeftVarOrder();
- setChildMetadata(leftChildNodeId, leftVarOrder, joinNodeId);
+ setChildMetadata(fluoQueryBuilder, leftChildNodeId, leftVarOrder, joinNodeId);
// Create or update the right child's variable order and parent node id.
final VariableOrder rightVarOrder = varOrders.getRightVarOrder();
- setChildMetadata(rightChildNodeId, rightVarOrder, joinNodeId);
+ setChildMetadata(fluoQueryBuilder, rightChildNodeId, rightVarOrder, joinNodeId);
}
@Override
@@ -407,7 +474,7 @@ public class SparqlFluoQueryBuilder {
// Update the child node's metadata.
final Set<String> childVars = getVars((TupleExpr)child);
final VariableOrder childVarOrder = new VariableOrder(childVars);
- setChildMetadata(childNodeId, childVarOrder, filterId);
+ setChildMetadata(fluoQueryBuilder, childNodeId, childVarOrder, filterId);
// Walk to the next node.
super.meet(node);
@@ -442,12 +509,12 @@ public class SparqlFluoQueryBuilder {
// Update the child node's metadata.
final Set<String> childVars = getVars((TupleExpr) child);
final VariableOrder childVarOrder = new VariableOrder(childVars);
- setChildMetadata(childNodeId, childVarOrder, periodicId);
+ setChildMetadata(fluoQueryBuilder, childNodeId, childVarOrder, periodicId);
// update variable order of this node and all ancestors to
// include BIN_ID binding as
// first variable in the ordering
- PeriodicQueryUtil.updateVarOrdersToIncludeBin(fluoQueryBuilder, periodicId);
+ FluoQueryUtils.updateVarOrders(fluoQueryBuilder, UpdateAction.AddVariable, Arrays.asList(IncrementalUpdateConstants.PERIODIC_BIN_ID), periodicId);
// Walk to the next node.
node.getArg().visit(this);
}
@@ -458,13 +525,12 @@ public class SparqlFluoQueryBuilder {
public void meet(final Projection node) {
// Create a builder for this node populated with the metadata.
final String queryId = nodeIds.getOrMakeId(node);
- final VariableOrder queryVarOrder = new VariableOrder(node.getBindingNames());
-
- final QueryMetadata.Builder queryBuilder = QueryMetadata.builder(queryId);
- fluoQueryBuilder.setQueryMetadata(queryBuilder);
- queryBuilder.setSparql(sparql);
- queryBuilder.setVariableOrder(queryVarOrder);
+ ProjectionMetadata.Builder projectionBuilder = fluoQueryBuilder.getProjectionBuilder(queryId).orNull();
+ if (projectionBuilder == null) {
+ projectionBuilder = ProjectionMetadata.builder(queryId);
+ fluoQueryBuilder.addProjectionBuilder(projectionBuilder);
+ }
final QueryModelNode child = node.getArg();
if(child == null) {
@@ -472,13 +538,14 @@ public class SparqlFluoQueryBuilder {
}
final String childNodeId = nodeIds.getOrMakeId(child);
- queryBuilder.setChildNodeId(childNodeId);
+ projectionBuilder.setChildNodeId(childNodeId);
+ projectionBuilder.setProjectedVars(projectionBuilder.getVariableOrder());
// Update the child node's metadata.
final Set<String> childVars = getVars((TupleExpr)child);
final VariableOrder childVarOrder = new VariableOrder(childVars);
- setChildMetadata(childNodeId, childVarOrder, queryId);
+ setChildMetadata(fluoQueryBuilder, childNodeId, childVarOrder, queryId);
// Walk to the next node.
super.meet(node);
@@ -489,10 +556,13 @@ public class SparqlFluoQueryBuilder {
//create id, initialize ConstructQueryMetadata builder, register ConstructQueryMetadata
//builder with FluoQueryBuilder, and add metadata that we currently have
final String constructId = nodeIds.getOrMakeId(node);
- final ConstructQueryMetadata.Builder constructBuilder = ConstructQueryMetadata.builder();
- constructBuilder.setNodeId(constructId);
- fluoQueryBuilder.setConstructQueryMetadata(constructBuilder);
- constructBuilder.setSparql(sparql);
+
+ ConstructQueryMetadata.Builder constructBuilder = fluoQueryBuilder.getConstructQueryBuilder().orNull();
+ if(constructBuilder == null) {
+ constructBuilder = ConstructQueryMetadata.builder();
+ constructBuilder.setNodeId(constructId);
+ fluoQueryBuilder.setConstructQueryMetadata(constructBuilder);
+ }
//get child node
QueryModelNode child = node.getArg();
@@ -531,96 +601,12 @@ public class SparqlFluoQueryBuilder {
// Update the child node's metadata.
final Set<String> childVars = getVars((TupleExpr)child);
final VariableOrder childVarOrder = new VariableOrder(childVars);
- setChildMetadata(childNodeId, childVarOrder, constructId);
+ setChildMetadata(fluoQueryBuilder, childNodeId, childVarOrder, constructId);
//fast forward visitor to next node we care about
child.visit(this);
}
-
- /**
- * Update a query node's metadata to include it's binding set variable order
- * and it's parent node id. This information is only known when handling
- * the parent node.
- *
- * @param childNodeId - The node ID of the child node.
- * @param childVarOrder - The variable order of the child node's binding sets.
- * @param parentNodeId - The node ID that consumes the child's binding sets.
- */
- private void setChildMetadata(final String childNodeId, final VariableOrder childVarOrder, final String parentNodeId) {
- checkNotNull(childNodeId);
- checkNotNull(childVarOrder);
- checkNotNull(parentNodeId);
-
- final NodeType childType = NodeType.fromNodeId(childNodeId).get();
- switch (childType) {
- case STATEMENT_PATTERN:
- StatementPatternMetadata.Builder spBuilder = fluoQueryBuilder.getStatementPatternBuilder(childNodeId).orNull();
- if (spBuilder == null) {
- spBuilder = StatementPatternMetadata.builder(childNodeId);
- fluoQueryBuilder.addStatementPatternBuilder(spBuilder);
- }
-
- spBuilder.setVarOrder(childVarOrder);
- spBuilder.setParentNodeId(parentNodeId);
- break;
-
- case JOIN:
- JoinMetadata.Builder joinBuilder = fluoQueryBuilder.getJoinBuilder(childNodeId).orNull();
- if (joinBuilder == null) {
- joinBuilder = JoinMetadata.builder(childNodeId);
- fluoQueryBuilder.addJoinMetadata(joinBuilder);
- }
-
- joinBuilder.setVariableOrder(childVarOrder);
- joinBuilder.setParentNodeId(parentNodeId);
- break;
-
- case FILTER:
- FilterMetadata.Builder filterBuilder = fluoQueryBuilder.getFilterBuilder(childNodeId).orNull();
- if (filterBuilder == null) {
- filterBuilder = FilterMetadata.builder(childNodeId);
- fluoQueryBuilder.addFilterMetadata(filterBuilder);
- }
-
- filterBuilder.setVarOrder(childVarOrder);
- filterBuilder.setParentNodeId(parentNodeId);
- break;
-
- case AGGREGATION:
- AggregationMetadata.Builder aggregationBuilder = fluoQueryBuilder.getAggregateBuilder(childNodeId).orNull();
- if (aggregationBuilder == null) {
- aggregationBuilder = AggregationMetadata.builder(childNodeId);
- fluoQueryBuilder.addAggregateMetadata(aggregationBuilder);
- }
-
- aggregationBuilder.setVariableOrder(childVarOrder);
- aggregationBuilder.setParentNodeId(parentNodeId);
- break;
-
- case QUERY:
- throw new IllegalArgumentException("A QUERY node cannot be the child of another node.");
-
- case CONSTRUCT:
- throw new IllegalArgumentException("A CONSTRUCT node cannot be the child of another node.");
-
- case PERIODIC_QUERY:
- PeriodicQueryMetadata.Builder periodicQueryBuilder = fluoQueryBuilder.getPeriodicQueryBuilder().orNull();
- if (periodicQueryBuilder == null) {
- periodicQueryBuilder = PeriodicQueryMetadata.builder();
- periodicQueryBuilder.setNodeId(childNodeId);
- fluoQueryBuilder.addPeriodicQueryMetadata(periodicQueryBuilder);
- }
- periodicQueryBuilder.setVarOrder(childVarOrder);
- periodicQueryBuilder.setParentNodeId(parentNodeId);
- break;
-
- default:
- throw new IllegalArgumentException("Unsupported NodeType: " + childType);
-
- }
- }
-
private ConstructGraph getConstructGraph(List<ProjectionElemList> projections, List<ExtensionElem> extensionElems) {
Map<String, Value> valueMap = new HashMap<>();
//create valueMap to associate source names with Values
@@ -654,6 +640,13 @@ public class SparqlFluoQueryBuilder {
return new ConstructGraph(constructProj);
}
+ private Set<String> getVarsToDelete(Collection<String> groupByVars, Collection<String> varOrderVars) {
+ Set<String> groupBySet = Sets.newHashSet(groupByVars);
+ Set<String> varOrderSet = Sets.newHashSet(varOrderVars);
+
+ return Sets.difference(varOrderSet, groupBySet);
+ }
+
private void validateProjectionElemList(ProjectionElemList list) {
List<ProjectionElem> elements = list.getElements();
checkArgument(elements.size() == 3);
@@ -662,8 +655,6 @@ public class SparqlFluoQueryBuilder {
checkArgument(elements.get(2).getTargetName().equals("object"));
}
-
-
/**
* Get the non-constant variables from a {@link TupleExpr}.
*
@@ -764,4 +755,199 @@ public class SparqlFluoQueryBuilder {
return shifted;
}
}
+
+ private void setVarOrderAndQueryType(QueryMetadata.Builder builder, TupleExpr te) {
+ QueryMetadataLocator locator = new QueryMetadataLocator();
+ try {
+ te.visit(locator);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ builder.setVarOrder(locator.getVarOrder());
+ builder.setQueryType(locator.getQueryType());
+ }
+
+ public static class QueryMetadataLocator extends QueryModelVisitorBase<Exception> {
+
+ private VariableOrder varOrder;
+ private QueryType queryType;
+
+ public VariableOrder getVarOrder() {
+ return varOrder;
+ }
+
+ public QueryType getQueryType() {
+ return queryType;
+ }
+
+ public void meet(Projection node) throws Exception {
+ Set<String> bindingNames = node.getBindingNames();
+ if(varOrder == null) {
+ varOrder = new VariableOrder(bindingNames);
+ }
+
+ if(queryType == null) {
+ queryType = QueryType.Projection;
+ }
+ super.meet(node);
+ }
+
+ public void meet(Reduced node) throws Exception {
+ if(varOrder == null) {
+ varOrder = getConstructGraphVarOrder(node);
+ }
+
+ if(queryType == null) {
+ queryType = QueryType.Construct;
+ }
+ super.meet(node);
+ }
+
+ public void meetOther(final QueryModelNode node) throws Exception {
+ if (node instanceof PeriodicQueryNode) {
+ queryType = QueryType.Periodic;
+ } else {
+ super.meetOther(node);
+ }
+ }
+ }
+
+ private static VariableOrder getConstructGraphVarOrder(Reduced node) {
+
+ //get child node
+ QueryModelNode child = node.getArg();
+ Preconditions.checkArgument(child instanceof Projection || child instanceof MultiProjection);
+ UnaryTupleOperator unary = (UnaryTupleOperator) child;
+
+ //get ProjectionElemList to build ConstructGraph
+ final List<ProjectionElemList> projections = new ArrayList<>();
+ if(unary instanceof Projection) {
+ projections.add(((Projection) unary).getProjectionElemList());
+ } else {
+ projections.addAll(((MultiProjection)unary).getProjections());
+ }
+
+ return getConstructGraphVarOrder(projections);
+ }
+
+ private static VariableOrder getConstructGraphVarOrder(List<ProjectionElemList> projections) {
+ Set<String> varOrders = new HashSet<>();
+
+ for(ProjectionElemList elems: projections) {
+ for(ProjectionElem elem: elems.getElements()) {
+ String name = elem.getSourceName();
+ if(!name.startsWith("-const-") && !name.startsWith("-anon-")) {
+ varOrders.add(name);
+ }
+ }
+ }
+
+ return new VariableOrder(varOrders);
+ }
+
+
+ /**
+ * Update a query node's metadata to include it's binding set variable order
+ * and it's parent node id. This information is only known when handling
+ * the parent node.
+ *
+ * @param fluoQueryBuilder - Builder whose metadata is updatad
+ * @param childNodeId - The node ID of the child node.
+ * @param childVarOrder - The variable order of the child node's binding sets.
+ * @param parentNodeId - The node ID that consumes the child's binding sets.
+ */
+ private static void setChildMetadata(final FluoQuery.Builder fluoQueryBuilder, final String childNodeId, final VariableOrder childVarOrder, final String parentNodeId) {
+ checkNotNull(childNodeId);
+ checkNotNull(childVarOrder);
+ checkNotNull(parentNodeId);
+
+ final NodeType childType = NodeType.fromNodeId(childNodeId).get();
+ switch (childType) {
+ case STATEMENT_PATTERN:
+ StatementPatternMetadata.Builder spBuilder = fluoQueryBuilder.getStatementPatternBuilder(childNodeId).orNull();
+ if (spBuilder == null) {
+ spBuilder = StatementPatternMetadata.builder(childNodeId);
+ fluoQueryBuilder.addStatementPatternBuilder(spBuilder);
+ }
+
+ spBuilder.setVarOrder(childVarOrder);
+ spBuilder.setParentNodeId(parentNodeId);
+ break;
+
+ case JOIN:
+ JoinMetadata.Builder joinBuilder = fluoQueryBuilder.getJoinBuilder(childNodeId).orNull();
+ if (joinBuilder == null) {
+ joinBuilder = JoinMetadata.builder(childNodeId);
+ fluoQueryBuilder.addJoinMetadata(joinBuilder);
+ }
+
+ joinBuilder.setVarOrder(childVarOrder);
+ joinBuilder.setParentNodeId(parentNodeId);
+ break;
+
+ case FILTER:
+ FilterMetadata.Builder filterBuilder = fluoQueryBuilder.getFilterBuilder(childNodeId).orNull();
+ if (filterBuilder == null) {
+ filterBuilder = FilterMetadata.builder(childNodeId);
+ fluoQueryBuilder.addFilterMetadata(filterBuilder);
+ }
+
+ filterBuilder.setVarOrder(childVarOrder);
+ filterBuilder.setParentNodeId(parentNodeId);
+ break;
+
+ case AGGREGATION:
+ AggregationMetadata.Builder aggregationBuilder = fluoQueryBuilder.getAggregateBuilder(childNodeId).orNull();
+ if (aggregationBuilder == null) {
+ aggregationBuilder = AggregationMetadata.builder(childNodeId);
+ fluoQueryBuilder.addAggregateMetadata(aggregationBuilder);
+ }
+
+ aggregationBuilder.setVarOrder(childVarOrder);
+ aggregationBuilder.setParentNodeId(parentNodeId);
+ break;
+
+ case PROJECTION:
+ ProjectionMetadata.Builder projectionBuilder = fluoQueryBuilder.getProjectionBuilder(childNodeId).orNull();
+ if(projectionBuilder == null) {
+ projectionBuilder = ProjectionMetadata.builder(childNodeId);
+ fluoQueryBuilder.addProjectionBuilder(projectionBuilder);
+ }
+
+ projectionBuilder.setVarOrder(childVarOrder);
+ projectionBuilder.setParentNodeId(parentNodeId);
+ break;
+
+ case QUERY:
+ throw new IllegalArgumentException("A QUERY node cannot be the child of another node.");
+
+ case CONSTRUCT:
+ ConstructQueryMetadata.Builder constructBuilder = fluoQueryBuilder.getConstructQueryBuilder().orNull();
+ if(constructBuilder == null) {
+ constructBuilder = ConstructQueryMetadata.builder();
+ constructBuilder.setNodeId(childNodeId);
+ fluoQueryBuilder.setConstructQueryMetadata(constructBuilder);
+ }
+
+ Preconditions.checkArgument(childNodeId.equals(constructBuilder.getNodeId()));
+ constructBuilder.setVarOrder(childVarOrder);
+ constructBuilder.setParentNodeId(parentNodeId);
+ break;
+
+ case PERIODIC_QUERY:
+ PeriodicQueryMetadata.Builder periodicQueryBuilder = fluoQueryBuilder.getPeriodicQueryBuilder().orNull();
+ if (periodicQueryBuilder == null) {
+ periodicQueryBuilder = PeriodicQueryMetadata.builder();
+ periodicQueryBuilder.setNodeId(childNodeId);
+ fluoQueryBuilder.addPeriodicQueryMetadata(periodicQueryBuilder);
+ }
+ periodicQueryBuilder.setVarOrder(childVarOrder);
+ periodicQueryBuilder.setParentNodeId(parentNodeId);
+ break;
+
+ default:
+ throw new IllegalArgumentException("Unsupported NodeType: " + childType);
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/StatementPatternMetadata.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/StatementPatternMetadata.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/StatementPatternMetadata.java
index 7de10d5..beead93 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/StatementPatternMetadata.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/StatementPatternMetadata.java
@@ -127,7 +127,7 @@ public class StatementPatternMetadata extends CommonNodeMetadata {
* Builds instances of {@link StatementPatternMetadata}.
*/
@DefaultAnnotation(NonNull.class)
- public static final class Builder {
+ public static final class Builder implements CommonNodeMetadata.Builder {
private final String nodeId;
private VariableOrder varOrder;
@@ -160,6 +160,11 @@ public class StatementPatternMetadata extends CommonNodeMetadata {
this.varOrder = varOrder;
return this;
}
+
+ @Override
+ public VariableOrder getVariableOrder() {
+ return varOrder;
+ }
/**
* Sets the statement pattern new statements are matched against.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/util/FluoQueryUtils.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/util/FluoQueryUtils.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/util/FluoQueryUtils.java
new file mode 100644
index 0000000..303f9bb
--- /dev/null
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/util/FluoQueryUtils.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.fluo.app.util;
+
+import java.util.List;
+
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
+import org.apache.rya.indexing.pcj.fluo.app.query.FluoQuery;
+import org.apache.rya.indexing.pcj.fluo.app.util.VariableOrderUpdateVisitor.UpdateAction;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Utility class for manipulating components of a {@link FluoQuery}.
+ *
+ */
+public class FluoQueryUtils {
+
+ /**
+ * Updates the {@link VariableOrder}s of a given {@link FluoQuery.Builder}.
+ * @param builder - builder whose VariableOrders will be updated
+ * @param action - add or delete variables
+ * @param variables - variables to be added or deleted
+ * @param stopNodeId - node to stop at
+ * @return - FluoQuery.Builder with updated VariableOrders
+ */
+ public static FluoQuery.Builder updateVarOrders(FluoQuery.Builder builder, UpdateAction action, List<String> variables, String stopNodeId) {
+ VariableOrderUpdateVisitor visitor = new VariableOrderUpdateVisitor(builder, action, variables, stopNodeId);
+ visitor.visit();
+
+ return builder;
+ }
+
+ /**
+ * Converts the fluo query id to a pcj id
+ * @param fluoQueryId - query id of the form query_prefix + _ + UUID
+ * @return the pcjid which consists of only the UUID portion of the fluo query id
+ */
+ public static String convertFluoQueryIdToPcjId(String fluoQueryId) {
+ Preconditions.checkNotNull(fluoQueryId);
+ String[] queryIdParts = fluoQueryId.split(IncrementalUpdateConstants.QUERY_PREFIX + "_");
+ Preconditions.checkArgument(queryIdParts.length == 2 && queryIdParts[1]!= null && queryIdParts[1].length() > 0);
+ return queryIdParts[1];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/util/PeriodicQueryUtil.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/util/PeriodicQueryUtil.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/util/PeriodicQueryUtil.java
index fd24af2..406ba4c 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/util/PeriodicQueryUtil.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/util/PeriodicQueryUtil.java
@@ -20,7 +20,6 @@ package org.apache.rya.indexing.pcj.fluo.app.util;
import static com.google.common.base.Preconditions.checkArgument;
-import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@@ -30,13 +29,8 @@ import org.apache.fluo.api.client.SnapshotBase;
import org.apache.fluo.api.data.Bytes;
import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
import org.apache.rya.indexing.pcj.fluo.app.NodeType;
-import org.apache.rya.indexing.pcj.fluo.app.query.AggregationMetadata;
-import org.apache.rya.indexing.pcj.fluo.app.query.FluoQuery;
import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns;
-import org.apache.rya.indexing.pcj.fluo.app.query.PeriodicQueryMetadata;
import org.apache.rya.indexing.pcj.fluo.app.query.PeriodicQueryNode;
-import org.apache.rya.indexing.pcj.fluo.app.query.QueryMetadata;
-import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
import org.openrdf.model.Literal;
import org.openrdf.model.URI;
import org.openrdf.model.Value;
@@ -194,72 +188,6 @@ public class PeriodicQueryUtil {
}
/**
- * Adds the variable "periodicBinId" to the beginning of all {@link VariableOrder}s for the
- * Metadata nodes that appear above the PeriodicQueryNode. This ensures that the binId is
- * written first in the Row so that bins can be easily scanned and deleted.
- * @param builder
- * @param nodeId
- */
- public static void updateVarOrdersToIncludeBin(FluoQuery.Builder builder, String nodeId) {
- NodeType type = NodeType.fromNodeId(nodeId).orNull();
- if (type == null) {
- throw new IllegalArgumentException("NodeId must be associated with an existing MetadataBuilder.");
- }
- switch (type) {
- case AGGREGATION:
- AggregationMetadata.Builder aggBuilder = builder.getAggregateBuilder(nodeId).orNull();
- if (aggBuilder != null) {
- VariableOrder varOrder = aggBuilder.getVariableOrder();
- VariableOrder groupOrder = aggBuilder.getGroupByVariableOrder();
- // update varOrder with BIN_ID
- List<String> orderList = new ArrayList<>(varOrder.getVariableOrders());
- orderList.add(0, IncrementalUpdateConstants.PERIODIC_BIN_ID);
- aggBuilder.setVariableOrder(new VariableOrder(orderList));
- // update groupVarOrder with BIN_ID
- List<String> groupOrderList = new ArrayList<>(groupOrder.getVariableOrders());
- groupOrderList.add(0, IncrementalUpdateConstants.PERIODIC_BIN_ID);
- aggBuilder.setGroupByVariableOrder(new VariableOrder(groupOrderList));
- // recursive call to update the VariableOrders of all ancestors
- // of this node
- updateVarOrdersToIncludeBin(builder, aggBuilder.getParentNodeId());
- } else {
- throw new IllegalArgumentException("There is no AggregationMetadata.Builder for the indicated Id.");
- }
- break;
- case PERIODIC_QUERY:
- PeriodicQueryMetadata.Builder periodicBuilder = builder.getPeriodicQueryBuilder().orNull();
- if (periodicBuilder != null && periodicBuilder.getNodeId().equals(nodeId)) {
- VariableOrder varOrder = periodicBuilder.getVarOrder();
- List<String> orderList = new ArrayList<>(varOrder.getVariableOrders());
- orderList.add(0, IncrementalUpdateConstants.PERIODIC_BIN_ID);
- periodicBuilder.setVarOrder(new VariableOrder(orderList));
- // recursive call to update the VariableOrders of all ancestors
- // of this node
- updateVarOrdersToIncludeBin(builder, periodicBuilder.getParentNodeId());
- } else {
- throw new IllegalArgumentException(
- "PeriodicQueryMetadata.Builder id does not match the indicated id. A query cannot have more than one PeriodicQueryMetadata Node.");
- }
- break;
- case QUERY:
- QueryMetadata.Builder queryBuilder = builder.getQueryBuilder().orNull();
- if (queryBuilder != null && queryBuilder.getNodeId().equals(nodeId)) {
- VariableOrder varOrder = queryBuilder.getVariableOrder();
- List<String> orderList = new ArrayList<>(varOrder.getVariableOrders());
- orderList.add(0, IncrementalUpdateConstants.PERIODIC_BIN_ID);
- queryBuilder.setVariableOrder(new VariableOrder(orderList));
- } else {
- throw new IllegalArgumentException(
- "QueryMetadata.Builder id does not match the indicated id. A query cannot have more than one QueryMetadata Node.");
- }
- break;
- default:
- throw new IllegalArgumentException(
- "Incorrectly positioned PeriodicQueryNode. The PeriodicQueryNode can only be positioned below Projections, Extensions, and ConstructQueryNodes.");
- }
- }
-
- /**
* Collects all Metadata node Ids that are ancestors of the PeriodicQueryNode and contain the variable
* {@link IncrementalUpdateConstants#PERIODIC_BIN_ID}.
* @param sx - Fluo Snapshot for scanning Fluo
@@ -277,6 +205,10 @@ public class PeriodicQueryUtil {
case PERIODIC_QUERY:
ids.add(nodeId);
break;
+ case PROJECTION:
+ ids.add(nodeId);
+ getPeriodicQueryNodeAncestorIds(sx, sx.get( Bytes.of(nodeId), FluoQueryColumns.PROJECTION_CHILD_NODE_ID).toString(), ids);
+ break;
case QUERY:
ids.add(nodeId);
getPeriodicQueryNodeAncestorIds(sx, sx.get(Bytes.of(nodeId), FluoQueryColumns.QUERY_CHILD_NODE_ID).toString(), ids);
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/util/VariableOrderUpdateVisitor.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/util/VariableOrderUpdateVisitor.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/util/VariableOrderUpdateVisitor.java
new file mode 100644
index 0000000..f433849
--- /dev/null
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/util/VariableOrderUpdateVisitor.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.fluo.app.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
+import org.apache.rya.indexing.pcj.fluo.app.query.AggregationMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.ConstructQueryMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.FilterMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.FluoQuery;
+import org.apache.rya.indexing.pcj.fluo.app.query.JoinMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.PeriodicQueryMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.ProjectionMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.QueryBuilderVisitorBase;
+import org.apache.rya.indexing.pcj.fluo.app.query.QueryMetadata;
+import org.apache.rya.indexing.pcj.fluo.app.query.StatementPatternMetadata;
+import org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Visitor that traverses a {@link FluoQuery.Builder} and performs the indicated {@link UpdateAction}
+ * on the {@link VariableOrder}s of each node using a provided list of variables. The visitor
+ * either adds the provided list of variables to the VariableOrder of each node or deletes the
+ * provided variables from the VariableOrder of each node.
+ *
+ */
+public class VariableOrderUpdateVisitor extends QueryBuilderVisitorBase {
+
+ /**
+ * Enum class indicating whether to add or delete variables from
+ * the VariableOrders of nodes in the FluoQuery.
+ *
+ */
+ public static enum UpdateAction {
+ AddVariable, DeleteVariable
+ };
+
+ private UpdateAction action;
+ private List<String> variables;
+ private String stopNodeId;
+
+ /**
+ * Creates a VariableOrderUpdateVisitor to update the variables in a given FluoQuery.Builder
+ * @param fluoBuilder - builder whose VariableOrder will be updated
+ * @param action - either add or delete
+ * @param variables - variables to be added or deleted
+ * @param stopNodeId - indicates the builder node to stop at
+ */
+ public VariableOrderUpdateVisitor(FluoQuery.Builder fluoBuilder, UpdateAction action, List<String> variables, String stopNodeId) {
+ super(fluoBuilder);
+ this.action = Preconditions.checkNotNull(action);
+ this.variables = Preconditions.checkNotNull(variables);
+ this.stopNodeId = Preconditions.checkNotNull(stopNodeId);
+ }
+
+ public void visit(QueryMetadata.Builder builder) {
+ builder.setVarOrder(updateOrder(builder.getVariableOrder()));
+ if(!atStopNode(builder.getNodeId())) {
+ super.visit(builder);
+ }
+ }
+
+ public void visit(ProjectionMetadata.Builder builder) {
+ builder.setVarOrder(updateOrder(builder.getVariableOrder()));
+ if(action == UpdateAction.AddVariable) {
+ builder.setProjectedVars(updateOrder(builder.getProjectionVars()));
+ }
+ if(!atStopNode(builder.getNodeId())) {
+ super.visit(builder);
+ }
+ }
+
+ public void visit(ConstructQueryMetadata.Builder builder) {
+ builder.setVarOrder(updateOrder(builder.getVariableOrder()));
+ if(!atStopNode(builder.getNodeId())) {
+ super.visit(builder);
+ }
+ }
+
+ public void visit(FilterMetadata.Builder builder) {
+ builder.setVarOrder(updateOrder(builder.getVariableOrder()));
+ if(!atStopNode(builder.getNodeId())) {
+ super.visit(builder);
+ }
+ }
+
+ public void visit(PeriodicQueryMetadata.Builder builder) {
+ builder.setVarOrder(updateOrder(builder.getVariableOrder()));
+ if(!atStopNode(builder.getNodeId())) {
+ super.visit(builder);
+ }
+ }
+
+ public void visit(JoinMetadata.Builder builder) {
+ builder.setVarOrder(updateOrder(builder.getVariableOrder()));
+ if(!atStopNode(builder.getNodeId())) {
+ super.visit(builder);
+ }
+ }
+
+ public void visit(AggregationMetadata.Builder builder) {
+ builder.setVarOrder(updateOrder(builder.getVariableOrder()));
+ builder.setGroupByVariableOrder(updateOrder(builder.getGroupByVariableOrder()));
+ if(!atStopNode(builder.getNodeId())) {
+ super.visit(builder);
+ }
+ }
+
+ public void visit(StatementPatternMetadata.Builder builder) {
+ if(!atStopNode(builder.getNodeId())) {
+ super.visit(builder);
+ }
+ }
+
+ boolean atStopNode(String nodeId) {
+ return nodeId.equals(stopNodeId);
+ }
+
+ private VariableOrder updateOrder(VariableOrder varOrder) {
+
+ switch (action) {
+ case AddVariable:
+ varOrder = addBindingToOrder(varOrder);
+ break;
+ case DeleteVariable:
+ varOrder = deleteBindingFromOrder(varOrder);
+ break;
+ }
+ return varOrder;
+ }
+
+ private VariableOrder addBindingToOrder(VariableOrder varOrder) {
+ List<String> orderList = new ArrayList<>(varOrder.getVariableOrders());
+ orderList.addAll(0, variables);
+ return new VariableOrder(orderList);
+ }
+
+ private VariableOrder deleteBindingFromOrder(VariableOrder varOrder) {
+ List<String> vars = new ArrayList<>();
+ varOrder.getVariableOrders().forEach(x -> {
+ if (!variables.contains(x) || x.equals(IncrementalUpdateConstants.PERIODIC_BIN_ID)) {
+ vars.add(x);
+ }
+ });
+ return new VariableOrder(vars);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/PeriodicQueryUtilTest.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/PeriodicQueryUtilTest.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/PeriodicQueryUtilTest.java
index c8ca6af..b40ba3f 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.app/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/PeriodicQueryUtilTest.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/PeriodicQueryUtilTest.java
@@ -26,7 +26,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants;
-import org.apache.rya.indexing.pcj.fluo.app.query.SparqlFluoQueryBuilder.NodeIds;
+import org.apache.rya.indexing.pcj.fluo.app.NodeType;
import org.apache.rya.indexing.pcj.fluo.app.util.PeriodicQueryUtil;
import org.apache.rya.indexing.pcj.fluo.app.util.PeriodicQueryUtil.PeriodicQueryNodeRelocator;
import org.apache.rya.indexing.pcj.fluo.app.util.PeriodicQueryUtil.PeriodicQueryNodeVisitor;
@@ -162,17 +162,17 @@ public class PeriodicQueryUtilTest {
+ "?obs <uri:hasTime> ?time. " //n
+ "?obs <uri:hasLattitude> ?lat }"; //n
- SPARQLParser parser = new SPARQLParser();
- ParsedQuery pq = parser.parseQuery(query, null);
SparqlFluoQueryBuilder builder = new SparqlFluoQueryBuilder();
- FluoQuery fluoQuery = builder.make(pq, new NodeIds());
+ builder.setSparql(query);
+ builder.setFluoQueryId(NodeType.generateNewFluoIdForType(NodeType.QUERY));
+ FluoQuery fluoQuery = builder.build();
PeriodicQueryMetadata periodicMeta = fluoQuery.getPeriodicQueryMetadata().orNull();
Assert.assertEquals(true, periodicMeta != null);
VariableOrder periodicVars = periodicMeta.getVariableOrder();
Assert.assertEquals(IncrementalUpdateConstants.PERIODIC_BIN_ID, periodicVars.getVariableOrders().get(0));
- QueryMetadata queryMeta = fluoQuery.getQueryMetadata().get();
+ QueryMetadata queryMeta = fluoQuery.getQueryMetadata();
VariableOrder queryVars = queryMeta.getVariableOrder();
Assert.assertEquals(IncrementalUpdateConstants.PERIODIC_BIN_ID, queryVars.getVariableOrders().get(0));
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryBuilderVisitorTest.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryBuilderVisitorTest.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryBuilderVisitorTest.java
new file mode 100644
index 0000000..b432868
--- /dev/null
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryBuilderVisitorTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.fluo.app.query;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.rya.indexing.pcj.fluo.app.NodeType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class QueryBuilderVisitorTest {
+
+ @Test
+ public void builderTest() {
+
+ FluoQuery.Builder fluoBuilder = FluoQuery.builder();
+
+ String queryId = NodeType.generateNewFluoIdForType(NodeType.QUERY);
+ String projectionId = NodeType.generateNewFluoIdForType(NodeType.PROJECTION);
+ String joinId = NodeType.generateNewFluoIdForType(NodeType.JOIN);
+ String leftSp = NodeType.generateNewFluoIdForType(NodeType.STATEMENT_PATTERN);
+ String rightSp = NodeType.generateNewFluoIdForType(NodeType.STATEMENT_PATTERN);
+
+ List<String> expected = Arrays.asList(queryId, projectionId, joinId, leftSp, rightSp);
+
+ QueryMetadata.Builder queryBuilder = QueryMetadata.builder(queryId);
+ queryBuilder.setChildNodeId(projectionId);
+
+ ProjectionMetadata.Builder projectionBuilder = ProjectionMetadata.builder(projectionId);
+ projectionBuilder.setChildNodeId(joinId);
+
+ JoinMetadata.Builder joinBuilder = JoinMetadata.builder(joinId);
+ joinBuilder.setLeftChildNodeId(leftSp);
+ joinBuilder.setRightChildNodeId(rightSp);
+
+ StatementPatternMetadata.Builder left = StatementPatternMetadata.builder(leftSp);
+ StatementPatternMetadata.Builder right = StatementPatternMetadata.builder(rightSp);
+
+ fluoBuilder.setQueryMetadata(queryBuilder);
+ fluoBuilder.addProjectionBuilder(projectionBuilder);
+ fluoBuilder.addJoinMetadata(joinBuilder);
+ fluoBuilder.addStatementPatternBuilder(left);
+ fluoBuilder.addStatementPatternBuilder(right);
+
+ QueryBuilderPrinter printer = new QueryBuilderPrinter(fluoBuilder);
+ printer.visit();
+ Assert.assertEquals(expected, printer.getIds());
+ }
+
+
+ public static class QueryBuilderPrinter extends QueryBuilderVisitorBase {
+
+ private List<String> ids = new ArrayList<>();
+
+ public List<String> getIds() {
+ return ids;
+ }
+
+ public QueryBuilderPrinter(FluoQuery.Builder builder) {
+ super(builder);
+ }
+
+ public void visit(QueryMetadata.Builder queryBuilder) {
+ System.out.println(queryBuilder.getNodeId());
+ ids.add(queryBuilder.getNodeId());
+ super.visit(queryBuilder);
+ }
+
+ public void visit(ProjectionMetadata.Builder projectionBuilder) {
+ System.out.println(projectionBuilder.getNodeId());
+ ids.add(projectionBuilder.getNodeId());
+ super.visit(projectionBuilder);
+ }
+
+ public void visit(JoinMetadata.Builder joinBuilder) {
+ System.out.println(joinBuilder.getNodeId());
+ ids.add(joinBuilder.getNodeId());
+ super.visit(joinBuilder);
+ }
+
+ public void visit(StatementPatternMetadata.Builder statementBuilder) {
+ System.out.println(statementBuilder.getNodeId());
+ ids.add(statementBuilder.getNodeId());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.app/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryMetadataVisitorTest.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryMetadataVisitorTest.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryMetadataVisitorTest.java
new file mode 100644
index 0000000..5c89a75
--- /dev/null
+++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/QueryMetadataVisitorTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.rya.indexing.pcj.fluo.app.query;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.rya.indexing.pcj.fluo.app.NodeType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class QueryMetadataVisitorTest {
+
+ @Test
+ public void builderTest() {
+ String query = "prefix function: <http://org.apache.rya/function#> " // n
+ + "prefix time: <http://www.w3.org/2006/time#> " // n
+ + "select ?id (count(?obs) as ?total) where {" // n
+ + "Filter(function:periodic(?time, 2, .5, time:hours)) " // n
+ + "?obs <uri:hasTime> ?time. " // n
+ + "?obs <uri:hasId> ?id } group by ?id"; // n
+
+ SparqlFluoQueryBuilder builder = new SparqlFluoQueryBuilder();
+ builder.setFluoQueryId(NodeType.generateNewFluoIdForType(NodeType.QUERY));
+ builder.setSparql(query);
+ FluoQuery fluoQuery = builder.build();
+
+ QueryMetadata queryMetadata = fluoQuery.getQueryMetadata();
+ String queryId = queryMetadata.getNodeId();
+ String projectionId = queryMetadata.getChildNodeId();
+ String aggId = fluoQuery.getProjectionMetadata(projectionId).get().getChildNodeId();
+ String periodicId = fluoQuery.getAggregationMetadata(aggId).get().getChildNodeId();
+ String joinId = fluoQuery.getPeriodicQueryMetadata(periodicId).get().getChildNodeId();
+ String leftSp = fluoQuery.getJoinMetadata(joinId).get().getLeftChildNodeId();
+ String rightSp = fluoQuery.getJoinMetadata(joinId).get().getRightChildNodeId();
+
+ List<String> expected = Arrays.asList(queryId, projectionId, aggId, periodicId, joinId, leftSp, rightSp);
+ QueryMetadataVisitor visitor = new QueryMetadataVisitor(fluoQuery);
+ visitor.visit();
+
+ Assert.assertEquals(expected, visitor.getIds());
+ }
+
+
+ public static class QueryMetadataVisitor extends QueryMetadataVisitorBase {
+
+ private List<String> ids = new ArrayList<>();
+
+ public List<String> getIds() {
+ return ids;
+ }
+
+ public QueryMetadataVisitor(FluoQuery metadata) {
+ super(metadata);
+ }
+
+ public void visit(QueryMetadata metadata) {
+ ids.add(metadata.getNodeId());
+ super.visit(metadata);
+ }
+
+ public void visit(ProjectionMetadata metadata) {
+ ids.add(metadata.getNodeId());
+ super.visit(metadata);
+ }
+
+ public void visit(JoinMetadata metadata) {
+ ids.add(metadata.getNodeId());
+ super.visit(metadata);
+ }
+
+ public void visit(StatementPatternMetadata metadata) {
+ ids.add(metadata.getNodeId());
+ }
+
+ public void visit(PeriodicQueryMetadata metadata) {
+ ids.add(metadata.getNodeId());
+ super.visit(metadata);
+ }
+
+ public void visit(FilterMetadata metadata) {
+ ids.add(metadata.getNodeId());
+ super.visit(metadata);
+ }
+
+ public void visit(AggregationMetadata metadata) {
+ ids.add(metadata.getNodeId());
+ super.visit(metadata);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/e387818b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/NewQueryCommand.java
----------------------------------------------------------------------
diff --git a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/NewQueryCommand.java b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/NewQueryCommand.java
index 854798d..3f335f4 100644
--- a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/NewQueryCommand.java
+++ b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/NewQueryCommand.java
@@ -41,7 +41,7 @@ import org.apache.logging.log4j.Logger;
import org.apache.rya.accumulo.AccumuloRdfConfiguration;
import org.apache.rya.accumulo.query.AccumuloRyaQueryEngine;
import org.apache.rya.api.persist.RyaDAOException;
-import org.apache.rya.indexing.pcj.fluo.api.CreatePcj;
+import org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj;
import org.apache.rya.indexing.pcj.fluo.client.PcjAdminClientCommand;
import org.apache.rya.indexing.pcj.fluo.client.util.ParsedQueryRequest;
import org.apache.rya.indexing.pcj.storage.PcjException;
@@ -124,7 +124,7 @@ public class NewQueryCommand implements PcjAdminClientCommand {
log.trace("SPARQL Query: " + request.getQuery());
log.trace("Var Orders: " + request.getVarOrders());
log.trace("Loading these values into the Fluo app.");
- final CreatePcj createPcj = new CreatePcj();
+ final CreateFluoPcj createPcj = new CreateFluoPcj();
try {
// Create the PCJ in Rya.
final String sparql = request.getQuery();