You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@rya.apache.org by mi...@apache.org on 2015/12/07 13:04:39 UTC
[09/51] [partial] incubator-rya git commit: Cannot delete temp branch,
doc'd it.
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectSpoTableOutput.java
----------------------------------------------------------------------
diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectSpoTableOutput.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectSpoTableOutput.java
deleted file mode 100644
index f7b1672..0000000
--- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectSpoTableOutput.java
+++ /dev/null
@@ -1,126 +0,0 @@
-package mvm.rya.joinselect.mr;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.AUTHS;
-import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.SPO_OUTPUTPATH;
-import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.SPO_TABLE;
-
-import java.io.IOException;
-
-import mvm.rya.accumulo.AccumuloRdfConfiguration;
-import mvm.rya.api.RdfCloudTripleStoreConstants;
-import mvm.rya.api.domain.RyaStatement;
-import mvm.rya.api.resolver.RyaTripleContext;
-import mvm.rya.api.resolver.triple.TripleRow;
-import mvm.rya.api.resolver.triple.TripleRowResolverException;
-import mvm.rya.joinselect.mr.utils.CompositeType;
-import mvm.rya.joinselect.mr.utils.JoinSelectStatsUtil;
-import mvm.rya.joinselect.mr.utils.TripleCard;
-import mvm.rya.joinselect.mr.utils.TripleEntry;
-
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.util.Tool;
-
-public class JoinSelectSpoTableOutput extends Configured implements Tool {
-
- public static class JoinSelectMapper extends Mapper<Key,Value,CompositeType,TripleCard> {
-
- private RyaTripleContext ryaContext;
- private static final String DELIM = "\u0000";
-
- public void map(Key row, Value data, Context context) throws IOException, InterruptedException {
- try {
- ryaContext = RyaTripleContext.getInstance(new AccumuloRdfConfiguration(context.getConfiguration()));
- RyaStatement ryaStatement = ryaContext.deserializeTriple(RdfCloudTripleStoreConstants.TABLE_LAYOUT.SPO, new TripleRow(row.getRow().getBytes(), row
- .getColumnFamily().getBytes(), row.getColumnQualifier().getBytes(), row.getTimestamp(), row.getColumnVisibility().getBytes(), data.get()));
-
- Text s = new Text(ryaStatement.getSubject().getData());
- Text p = new Text(ryaStatement.getPredicate().getData());
- Text o = new Text(ryaStatement.getObject().getData());
- Text sp = new Text(ryaStatement.getSubject().getData() + DELIM + ryaStatement.getPredicate().getData());
- Text po = new Text(ryaStatement.getPredicate().getData() + DELIM + ryaStatement.getObject().getData());
- Text so = new Text(ryaStatement.getSubject().getData() + DELIM + ryaStatement.getObject().getData());
- Text ps = new Text(ryaStatement.getPredicate().getData() + DELIM + ryaStatement.getSubject().getData());
- Text op = new Text(ryaStatement.getObject().getData() + DELIM + ryaStatement.getPredicate().getData());
- Text os = new Text(ryaStatement.getObject().getData() + DELIM + ryaStatement.getSubject().getData());
-
- TripleEntry t1 = new TripleEntry(s, p, new Text("subject"), new Text("predicate"), new Text("object"));
- TripleEntry t2 = new TripleEntry(p, o, new Text("predicate"), new Text("object"), new Text("subject"));
- TripleEntry t3 = new TripleEntry(o, s, new Text("object"), new Text("subject"), new Text("predicate"));
- TripleEntry t4 = new TripleEntry(s, new Text(""), new Text("subject"), new Text(""), new Text("predicateobject"));
- TripleEntry t5 = new TripleEntry(p, new Text(""), new Text("predicate"), new Text(""), new Text("objectsubject"));
- TripleEntry t6 = new TripleEntry(o, new Text(""), new Text("object"), new Text(""), new Text("subjectpredicate"));
- TripleEntry t7 = new TripleEntry(s, new Text(""), new Text("subject"), new Text(""), new Text("objectpredicate"));
- TripleEntry t8 = new TripleEntry(p, new Text(""), new Text("predicate"), new Text(""), new Text("subjectobject"));
- TripleEntry t9 = new TripleEntry(o, new Text(""), new Text("object"), new Text(""), new Text("predicatesubject"));
-
- context.write(new CompositeType(o, new IntWritable(2)), new TripleCard(t1));
- context.write(new CompositeType(s, new IntWritable(2)), new TripleCard(t2));
- context.write(new CompositeType(p, new IntWritable(2)), new TripleCard(t3));
- context.write(new CompositeType(po, new IntWritable(2)), new TripleCard(t4));
- context.write(new CompositeType(so, new IntWritable(2)), new TripleCard(t5));
- context.write(new CompositeType(sp, new IntWritable(2)), new TripleCard(t6));
- context.write(new CompositeType(op, new IntWritable(2)), new TripleCard(t7));
- context.write(new CompositeType(os, new IntWritable(2)), new TripleCard(t8));
- context.write(new CompositeType(ps, new IntWritable(2)), new TripleCard(t9));
-
- } catch (TripleRowResolverException e) {
- e.printStackTrace();
- }
-
- }
-
- }
-
- @Override
- public int run(String[] args) throws Exception {
-
- Configuration conf = getConf();
- String inTable = conf.get(SPO_TABLE);
- String auths = conf.get(AUTHS);
- String outPath = conf.get(SPO_OUTPUTPATH);
-
- assert inTable != null && outPath != null;
-
- Job job = new Job(conf, this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
- job.setJarByClass(this.getClass());
- conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
-
- JoinSelectStatsUtil.initTabToSeqFileJob(job, inTable, outPath, auths);
- job.setMapperClass(JoinSelectMapper.class);
- job.setNumReduceTasks(0);
- job.waitForCompletion(true);
-
- return job.isSuccessful() ? 0 : 1;
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectStatisticsSum.java
----------------------------------------------------------------------
diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectStatisticsSum.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectStatisticsSum.java
deleted file mode 100644
index ef271ff..0000000
--- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectStatisticsSum.java
+++ /dev/null
@@ -1,220 +0,0 @@
-package mvm.rya.joinselect.mr;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.AUTHS;
-import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.INPUTPATH;
-import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.SELECTIVITY_TABLE;
-
-import java.io.IOException;
-
-import mvm.rya.joinselect.mr.utils.CardList;
-import mvm.rya.joinselect.mr.utils.JoinSelectStatsUtil;
-import mvm.rya.joinselect.mr.utils.TripleEntry;
-
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.util.Tool;
-
-public class JoinSelectStatisticsSum extends Configured implements Tool {
-
- // TODO need to tweak this class to compute join cardinalities over more than one variable
-
- public static class CardinalityIdentityMapper extends Mapper<TripleEntry,CardList,TripleEntry,CardList> {
-
- public void map(TripleEntry key, CardList value, Context context) throws IOException, InterruptedException {
-
- // System.out.println("Keys are " + key + " and values are " + value);
-
- if (key.getSecond().toString().length() != 0 && key.getSecondPos().toString().length() != 0) {
- TripleEntry te1 = new TripleEntry(key.getFirst(), new Text(""), key.getFirstPos(), new Text(""), key.getKeyPos());
- TripleEntry te2 = new TripleEntry(key.getSecond(), new Text(""), key.getSecondPos(), new Text(""), key.getKeyPos());
-
- context.write(te1, value);
- context.write(te2, value);
- context.write(key, value);
- // System.out.println("Output key values from mapper are " + te1 + " and " + value + "\n"
- // + te2 + " and " + value + "\n" + key + " and " + value + "\n");
- } else if (key.getSecond().toString().length() == 0 && key.getSecondPos().toString().length() == 0) {
-
- context.write(key, value);
- // System.out.println("Output key values from mapper are " + "\n" + key + " and " + value + "\n" + "\n");
- }
-
- }
-
- }
-
- public static class CardinalityIdentityReducer extends Reducer<TripleEntry,CardList,Text,Mutation> {
-
- private static final String DELIM = "\u0000";
-
- public void reduce(TripleEntry te, Iterable<CardList> values, Context context) throws IOException, InterruptedException {
-
- CardList cl = new CardList();
- LongWritable s = new LongWritable(0);
- LongWritable p = new LongWritable(0);
- LongWritable o = new LongWritable(0);
- LongWritable sp = new LongWritable(0);
- LongWritable po = new LongWritable(0);
- LongWritable so = new LongWritable(0);
-
- // System.out.println("***********************************************************\n"
- // + "key is " + te);
-
- for (CardList val : values) {
- // System.out.println("Value is " + val);
- s.set(s.get() + val.getcardS().get());
- p.set(p.get() + val.getcardP().get());
- o.set(o.get() + val.getcardO().get());
- sp.set(sp.get() + val.getcardSP().get());
- po.set(po.get() + val.getcardPO().get());
- so.set(so.get() + val.getcardSO().get());
- }
- cl.setCard(s, p, o, sp, po, so);
-
- Text row;
-
- if (te.getSecond().toString().length() > 0) {
- row = new Text(te.getFirstPos().toString() + te.getSecondPos().toString() + DELIM + te.getFirst().toString() + DELIM + te.getSecond());
- } else {
- row = new Text(te.getFirstPos().toString() + DELIM + te.getFirst().toString());
- }
-
- Mutation m1, m2, m3;
-
- if (te.getKeyPos().toString().equals("subject") || te.getKeyPos().toString().equals("predicate") || te.getKeyPos().toString().equals("object")) {
- m1 = new Mutation(row);
- m1.put(new Text(te.getKeyPos().toString() + "subject"), new Text(cl.getcardS().toString()), new Value(new byte[0]));
- m2 = new Mutation(row);
- m2.put(new Text(te.getKeyPos().toString() + "predicate"), new Text(cl.getcardP().toString()), new Value(new byte[0]));
- m3 = new Mutation(row);
- m3.put(new Text(te.getKeyPos().toString() + "object"), new Text(cl.getcardO().toString()), new Value(new byte[0]));
-
- } else if (te.getKeyPos().toString().equals("predicatesubject") || te.getKeyPos().toString().equals("objectpredicate")
- || te.getKeyPos().toString().equals("subjectobject")) {
-
- String jOrder = reverseJoinOrder(te.getKeyPos().toString());
-
- m1 = new Mutation(row);
- m1.put(new Text(jOrder + "predicatesubject"), new Text(cl.getcardSP().toString()), new Value(new byte[0]));
- m2 = new Mutation(row);
- m2.put(new Text(jOrder + "objectpredicate"), new Text(cl.getcardPO().toString()), new Value(new byte[0]));
- m3 = new Mutation(row);
- m3.put(new Text(jOrder + "subjectobject"), new Text(cl.getcardSO().toString()), new Value(new byte[0]));
-
- } else {
-
- m1 = new Mutation(row);
- m1.put(new Text(te.getKeyPos().toString() + "subjectpredicate"), new Text(cl.getcardSP().toString()), new Value(new byte[0]));
- m2 = new Mutation(row);
- m2.put(new Text(te.getKeyPos().toString() + "predicateobject"), new Text(cl.getcardPO().toString()), new Value(new byte[0]));
- m3 = new Mutation(row);
- m3.put(new Text(te.getKeyPos().toString() + "objectsubject"), new Text(cl.getcardSO().toString()), new Value(new byte[0]));
-
- }
-
- // TODO add the appropriate table name here
- context.write(new Text(""), m1);
- context.write(new Text(""), m2);
- context.write(new Text(""), m3);
- }
-
- private String reverseJoinOrder(String s) {
-
- if (s.equals("predicatesubject")) {
- return "subjectpredicate";
- } else if (s.equals("objectpredicate")) {
- return "predicateobject";
- } else if (s.equals("subjectobject")) {
- return "objectsubject";
- } else {
- throw new IllegalArgumentException("Invalid join type.");
- }
-
- }
-
- }
-
- public static class CardinalityIdentityCombiner extends Reducer<TripleEntry,CardList,TripleEntry,CardList> {
-
- @Override
- public void reduce(TripleEntry key, Iterable<CardList> values, Context context) throws IOException, InterruptedException {
-
- CardList cl = new CardList();
- LongWritable s = new LongWritable(0);
- LongWritable p = new LongWritable(0);
- LongWritable o = new LongWritable(0);
- LongWritable sp = new LongWritable(0);
- LongWritable po = new LongWritable(0);
- LongWritable so = new LongWritable(0);
-
- for (CardList val : values) {
- s.set(s.get() + val.getcardS().get());
- p.set(p.get() + val.getcardP().get());
- o.set(o.get() + val.getcardO().get());
- sp.set(sp.get() + val.getcardSP().get());
- po.set(po.get() + val.getcardPO().get());
- so.set(so.get() + val.getcardSO().get());
- }
-
- cl.setCard(s, p, o, sp, po, so);
- context.write(key, cl);
-
- }
-
- }
-
- @Override
- public int run(String[] args) throws AccumuloSecurityException, IOException, ClassNotFoundException, InterruptedException {
-
- Configuration conf = getConf();
- String outTable = conf.get(SELECTIVITY_TABLE);
- String auths = conf.get(AUTHS);
- String inPath = conf.get(INPUTPATH);
-
- assert outTable != null && inPath != null;
-
- Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
- job.setJarByClass(this.getClass());
- JoinSelectStatsUtil.initSumMRJob(job, inPath, outTable, auths);
-
- job.setMapperClass(CardinalityIdentityMapper.class);
- job.setCombinerClass(CardinalityIdentityCombiner.class);
- job.setReducerClass(CardinalityIdentityReducer.class);
- job.setNumReduceTasks(32);
-
- job.waitForCompletion(true);
-
- return job.isSuccessful() ? 0 : 1;
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardList.java
----------------------------------------------------------------------
diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardList.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardList.java
deleted file mode 100644
index b8fd274..0000000
--- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardList.java
+++ /dev/null
@@ -1,209 +0,0 @@
-package mvm.rya.joinselect.mr.utils;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.WritableComparable;
-
-public class CardList implements WritableComparable<CardList> {
-
- private LongWritable cardS;
- private LongWritable cardP;
- private LongWritable cardO;
- private LongWritable cardSP;
- private LongWritable cardPO;
- private LongWritable cardSO;
-
- public CardList() {
- cardS = new LongWritable();
- cardP = new LongWritable();
- cardO = new LongWritable();
- cardSP = new LongWritable();
- cardSO = new LongWritable();
- cardPO = new LongWritable();
-
- }
-
- public CardList(long cardS, long cardP, long cardO, long cardSP, long cardPO, long cardSO) {
- this.cardS = new LongWritable(cardS);
- this.cardP = new LongWritable(cardP);
- this.cardO = new LongWritable(cardO);
- this.cardSP = new LongWritable(cardSP);
- this.cardSO = new LongWritable(cardSO);
- this.cardPO = new LongWritable(cardPO);
- }
-
- public CardList(LongWritable cardS, LongWritable cardP, LongWritable cardO, LongWritable cardSP, LongWritable cardPO, LongWritable cardSO) {
-
- this.cardS = cardS;
- this.cardP = cardP;
- this.cardO = cardO;
- this.cardSP = cardSP;
- this.cardPO = cardPO;
- this.cardSO = cardSO;
-
- }
-
- public void setCard(LongWritable cardS, LongWritable cardP, LongWritable cardO, LongWritable cardSP, LongWritable cardPO, LongWritable cardSO) {
- this.cardS = cardS;
- this.cardP = cardP;
- this.cardO = cardO;
- this.cardSP = cardSP;
- this.cardPO = cardPO;
- this.cardSO = cardSO;
-
- }
-
- public void setSCard(long cardS) {
- this.cardS = new LongWritable(cardS);
- }
-
- public void setPCard(long cardP) {
- this.cardP = new LongWritable(cardP);
- }
-
- public void setOCard(long cardO) {
- this.cardO = new LongWritable(cardO);
- }
-
- public void setSPCard(long cardSP) {
- this.cardSP = new LongWritable(cardSP);
- }
-
- public void setSOCard(long cardSO) {
- this.cardSO = new LongWritable(cardSO);
- }
-
- public void setPOCard(long cardPO) {
- this.cardPO = new LongWritable(cardPO);
- }
-
- public LongWritable getcardS() {
- return this.cardS;
- }
-
- public LongWritable getcardP() {
- return this.cardP;
- }
-
- public LongWritable getcardO() {
- return this.cardO;
- }
-
- public LongWritable getcardPO() {
- return this.cardPO;
- }
-
- public LongWritable getcardSO() {
- return this.cardSO;
- }
-
- public LongWritable getcardSP() {
- return this.cardSP;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- cardS.write(out);
- cardP.write(out);
- cardO.write(out);
- cardSO.write(out);
- cardPO.write(out);
- cardSP.write(out);
-
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- cardS.readFields(in);
- cardP.readFields(in);
- cardO.readFields(in);
- cardSO.readFields(in);
- cardPO.readFields(in);
- cardSP.readFields(in);
-
- }
-
- @Override
- public int hashCode() {
- int result = 7;
- result = result * 17 + cardS.hashCode();
- result = result * 17 + cardP.hashCode();
- result = result * 17 + cardO.hashCode();
- result = result * 17 + cardSP.hashCode();
- result = result * 17 + cardPO.hashCode();
- result = result * 17 + cardSO.hashCode();
-
- return result;
-
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof CardList) {
- CardList comp = (CardList) o;
- return cardS.equals(comp.cardS) && cardP.equals(comp.cardP) && cardO.equals(comp.cardO) && cardSP.equals(comp.cardSP) && cardSO.equals(comp.cardSO)
- && cardPO.equals(comp.cardPO);
-
- }
- return false;
- }
-
- @Override
- public String toString() {
- return cardS + "\t" + cardP + "\t" + cardO + "\t" + cardSP + "\t" + cardPO + "\t" + cardSO;
-
- }
-
- @Override
- public int compareTo(CardList o) {
-
- int cmp = cardS.compareTo(o.cardS);
- if (cmp != 0) {
- return cmp;
- }
- cmp = cardP.compareTo(o.cardP);
- if (cmp != 0) {
- return cmp;
- }
- cmp = cardO.compareTo(o.cardO);
- if (cmp != 0) {
- return cmp;
- }
- cmp = cardSP.compareTo(o.cardSP);
- if (cmp != 0) {
- return cmp;
- }
- cmp = cardPO.compareTo(o.cardPO);
- if (cmp != 0) {
- return cmp;
- }
-
- return cardSO.compareTo(o.cardSO);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardinalityType.java
----------------------------------------------------------------------
diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardinalityType.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardinalityType.java
deleted file mode 100644
index 924f596..0000000
--- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardinalityType.java
+++ /dev/null
@@ -1,149 +0,0 @@
-package mvm.rya.joinselect.mr.utils;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
-
-public class CardinalityType implements WritableComparable<CardinalityType> {
-
- private LongWritable card;
- private Text cardType;
- private LongWritable ts;
-
- public CardinalityType() {
- card = new LongWritable();
- cardType = new Text();
- ts = new LongWritable();
- }
-
- public CardinalityType(int card, String cardType, long ts) {
-
- this.card = new LongWritable(card);
- this.cardType = new Text(cardType);
- this.ts = new LongWritable(ts);
-
- }
-
- public CardinalityType(LongWritable card, Text cardType, LongWritable ts) {
-
- this.card = card;
- this.ts = ts;
- this.cardType = cardType;
-
- }
-
- public void set(CardinalityType ct) {
- this.card.set(ct.card.get());
- this.ts.set(ct.ts.get());
- this.cardType.set(ct.cardType);
- }
-
- public void setCard(LongWritable card) {
- this.card = card;
-
- }
-
- public void setCardType(Text cardType) {
- this.cardType = cardType;
- }
-
- public void setTS(LongWritable ts) {
- this.ts = ts;
- }
-
- public LongWritable getCard() {
- return this.card;
- }
-
- public Text getCardType() {
- return this.cardType;
- }
-
- public LongWritable getTS() {
- return this.ts;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- card.write(out);
- cardType.write(out);
- ts.write(out);
-
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- card.readFields(in);
- cardType.readFields(in);
- ts.readFields(in);
-
- }
-
- @Override
- public int hashCode() {
- int result = 7;
- result = result * 17 + card.hashCode();
- result = result * 17 + cardType.hashCode();
- result = result * 17 + ts.hashCode();
-
- return result;
-
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof CardinalityType) {
- CardinalityType trip = (CardinalityType) o;
- return card.equals(trip.card) && cardType.equals(trip.cardType) && ts.equals(trip.ts);
-
- }
- return false;
- }
-
- @Override
- public String toString() {
- return card + " " + cardType + " " + ts;
-
- }
-
- @Override
- public int compareTo(CardinalityType o) {
-
- int cmp = cardType.compareTo(o.cardType);
- if (cmp != 0) {
- return cmp;
- }
- cmp = ts.compareTo(o.ts);
- if (cmp != 0) {
- return cmp;
- }
- return card.compareTo(o.card);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CompositeType.java
----------------------------------------------------------------------
diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CompositeType.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CompositeType.java
deleted file mode 100644
index 57e6ee2..0000000
--- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CompositeType.java
+++ /dev/null
@@ -1,122 +0,0 @@
-package mvm.rya.joinselect.mr.utils;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
-
-public class CompositeType implements WritableComparable<CompositeType> {
-
- private Text oldKey;
- private IntWritable priority;
-
- public CompositeType() {
- oldKey = new Text();
- priority = new IntWritable();
- }
-
- public CompositeType(String oldKey, int priority) {
- this.oldKey = new Text(oldKey);
- this.priority = new IntWritable(priority);
- }
-
- public CompositeType(Text oldKey, IntWritable priority) {
-
- this.oldKey = oldKey;
- this.priority = priority;
-
- }
-
- public void setOldKey(Text oldKey) {
- this.oldKey = oldKey;
-
- }
-
- public void setPriority(IntWritable priority) {
- this.priority = priority;
- }
-
- public Text getOldKey() {
- return this.oldKey;
- }
-
- public IntWritable getPriority() {
- return this.priority;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- oldKey.write(out);
- priority.write(out);
-
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- oldKey.readFields(in);
- priority.readFields(in);
-
- }
-
- @Override
- public int hashCode() {
- int result = 7;
- result = result * 17 + oldKey.hashCode();
- // result = result*17+ priority.hashCode();
-
- return result;
-
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof CompositeType) {
- CompositeType comp = (CompositeType) o;
- return oldKey.equals(comp.oldKey) && priority.equals(comp.priority);
-
- }
- return false;
- }
-
- @Override
- public String toString() {
- return oldKey + "\t" + priority;
-
- }
-
- @Override
- public int compareTo(CompositeType o) {
- int compare = getOldKey().compareTo(o.getOldKey());
- if (compare != 0) {
- return compare;
- }
-
- return getPriority().compareTo(o.getPriority());
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectConstants.java
----------------------------------------------------------------------
diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectConstants.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectConstants.java
deleted file mode 100644
index 3ec34d0..0000000
--- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectConstants.java
+++ /dev/null
@@ -1,46 +0,0 @@
-package mvm.rya.joinselect.mr.utils;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-public class JoinSelectConstants {
-
- public static final String COUNT = "count";
- public static final String METADATA = "metadata";
- public static final byte[] EMPTY = new byte[0];
-
- // config properties
- public static final String PERFORMANT = "performant";
- public static final String USERNAME = "username";
- public static final String PASSWORD = "password";
- public static final String INSTANCE = "instance";
- public static final String ZOOKEEPERS = "zookeepers";
- public static final String INPUTPATH = "inputpath";
- public static final String OUTPUTPATH = "outputpath";
- public static final String PROSPECTS_OUTPUTPATH = "prospects.outputpath";
- public static final String SPO_OUTPUTPATH = "spo.outputpath";
- public static final String AUTHS = "auths";
- public static final String PROSPECTS_TABLE = "prospects.table";
- public static final String SPO_TABLE = "spo.table";
- public static final String SELECTIVITY_TABLE = "selectivity.table";
- public static final String MOCK = "mock";
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectStatsUtil.java
----------------------------------------------------------------------
diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectStatsUtil.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectStatsUtil.java
deleted file mode 100644
index cf8db40..0000000
--- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectStatsUtil.java
+++ /dev/null
@@ -1,183 +0,0 @@
-package mvm.rya.joinselect.mr.utils;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import static mvm.rya.accumulo.AccumuloRdfConstants.EMPTY_CV;
-import static mvm.rya.accumulo.AccumuloRdfConstants.EMPTY_VALUE;
-import static mvm.rya.api.RdfCloudTripleStoreConstants.EMPTY_TEXT;
-import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.INSTANCE;
-import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.PASSWORD;
-import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.USERNAME;
-import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.ZOOKEEPERS;
-
-import java.io.IOException;
-
-import mvm.rya.api.resolver.triple.TripleRow;
-
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
-import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.input.MultipleInputs;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-
-public class JoinSelectStatsUtil {
-
- public static void initSumMRJob(Job job, String inputPath, String outtable, String auths) throws AccumuloSecurityException, IOException {
- Configuration conf = job.getConfiguration();
- String username = conf.get(USERNAME);
- String password = conf.get(PASSWORD);
- String instance = conf.get(INSTANCE);
- String zookeepers = conf.get(ZOOKEEPERS);
-
- if (zookeepers != null) {
- AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password));
- AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers);
- } else {
- throw new IllegalArgumentException("Must specify zookeepers");
- }
-
- SequenceFileInputFormat.addInputPath(job, new Path(inputPath));
- job.setInputFormatClass(SequenceFileInputFormat.class);
- job.setMapOutputKeyClass(TripleEntry.class);
- job.setMapOutputValueClass(CardList.class);
-
- AccumuloOutputFormat.setDefaultTableName(job, outtable);
- job.setOutputFormatClass(AccumuloOutputFormat.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(Mutation.class);
-
- }
-
- public static void initTableMRJob(Job job, String intable, String outtable, String auths) throws AccumuloSecurityException {
- Configuration conf = job.getConfiguration();
- String username = conf.get(USERNAME);
- String password = conf.get(PASSWORD);
- String instance = conf.get(INSTANCE);
- String zookeepers = conf.get(ZOOKEEPERS);
-
- System.out.println("Zookeepers are " + auths);
-
- if (zookeepers != null) {
- AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers);
- AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers);
- } else {
- throw new IllegalArgumentException("Must specify either mock or zookeepers");
- }
-
- AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password));
- AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths));
- AccumuloInputFormat.setInputTableName(job, intable);
- job.setInputFormatClass(AccumuloInputFormat.class);
- job.setMapOutputKeyClass(Text.class);
- job.setMapOutputValueClass(IntWritable.class);
-
- // OUTPUT
- AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password));
- AccumuloOutputFormat.setDefaultTableName(job, outtable);
- job.setOutputFormatClass(AccumuloOutputFormat.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(Mutation.class);
-
- }
-
- public static void initTabToSeqFileJob(Job job, String intable, String outpath, String auths) throws AccumuloSecurityException {
-
- Configuration conf = job.getConfiguration();
- String username = conf.get(USERNAME);
- String password = conf.get(PASSWORD);
- String instance = conf.get(INSTANCE);
- String zookeepers = conf.get(ZOOKEEPERS);
-
- System.out.println("Zookeepers are " + auths);
-
- if (zookeepers != null) {
- AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers);
- } else {
- throw new IllegalArgumentException("Must specify either mock or zookeepers");
- }
-
- AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password));
- AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths));
- AccumuloInputFormat.setInputTableName(job, intable);
- job.setInputFormatClass(AccumuloInputFormat.class);
- job.setMapOutputKeyClass(CompositeType.class);
- job.setMapOutputValueClass(TripleCard.class);
-
- // OUTPUT
- SequenceFileOutputFormat.setOutputPath(job, new Path(outpath));
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setOutputKeyClass(CompositeType.class);
- job.setOutputValueClass(TripleCard.class);
-
- }
-
- public static void initJoinMRJob(Job job, String prospectsPath, String spoPath, Class<? extends Mapper<CompositeType,TripleCard,?,?>> mapperClass,
- String outPath, String auths) throws AccumuloSecurityException {
-
- MultipleInputs.addInputPath(job, new Path(prospectsPath), SequenceFileInputFormat.class, mapperClass);
- MultipleInputs.addInputPath(job, new Path(spoPath), SequenceFileInputFormat.class, mapperClass);
- job.setMapOutputKeyClass(CompositeType.class);
- job.setMapOutputValueClass(TripleCard.class);
-
- SequenceFileOutputFormat.setOutputPath(job, new Path(outPath));
- job.setOutputFormatClass(SequenceFileOutputFormat.class);
- job.setOutputKeyClass(TripleEntry.class);
- job.setOutputValueClass(CardList.class);
-
- }
-
- public static Mutation createMutation(TripleRow tripleRow) {
- Mutation mutation = new Mutation(new Text(tripleRow.getRow()));
- byte[] columnVisibility = tripleRow.getColumnVisibility();
- ColumnVisibility cv = columnVisibility == null ? EMPTY_CV : new ColumnVisibility(columnVisibility);
- Long timestamp = tripleRow.getTimestamp();
- boolean hasts = timestamp != null;
- timestamp = timestamp == null ? 0l : timestamp;
- byte[] value = tripleRow.getValue();
- Value v = value == null ? EMPTY_VALUE : new Value(value);
- byte[] columnQualifier = tripleRow.getColumnQualifier();
- Text cqText = columnQualifier == null ? EMPTY_TEXT : new Text(columnQualifier);
- byte[] columnFamily = tripleRow.getColumnFamily();
- Text cfText = columnFamily == null ? EMPTY_TEXT : new Text(columnFamily);
-
- if (hasts) {
- mutation.put(cfText, cqText, cv, timestamp, v);
- } else {
- mutation.put(cfText, cqText, cv, v);
-
- }
- return mutation;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleCard.java
----------------------------------------------------------------------
diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleCard.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleCard.java
deleted file mode 100644
index 467f754..0000000
--- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleCard.java
+++ /dev/null
@@ -1,145 +0,0 @@
-package mvm.rya.joinselect.mr.utils;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.WritableComparable;
-
-public class TripleCard implements WritableComparable<TripleCard> {
-
- private CardinalityType card = null;
- private TripleEntry te = null;
-
- private CardinalityType tempCard = new CardinalityType();
- private TripleEntry tempTe = new TripleEntry();
-
- public TripleCard() {}
-
- public TripleCard(CardinalityType card) {
- this.setCard(card);
- }
-
- public TripleCard(TripleEntry te) {
- this.setTE(te);
- }
-
- public void setCard(CardinalityType card) {
- tempCard.set(card);
- this.card = tempCard;
- this.te = null;
- }
-
- public void setTE(TripleEntry te) {
- tempTe.setTE(te);
- this.te = tempTe;
- this.card = null;
- }
-
- public CardinalityType getCard() {
- return this.card;
- }
-
- public TripleEntry getTE() {
- return this.te;
- }
-
- public boolean isCardNull() {
- return (card == null);
- }
-
- public boolean isTeNull() {
- return (te == null);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- if (card != null) {
- out.writeBoolean(true);
- card.write(out);
- } else {
- out.writeBoolean(false);
- te.write(out);
- }
-
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- if (in.readBoolean()) {
- tempCard.readFields(in);
- card = tempCard;
- te = null;
- } else {
- tempTe.readFields(in);
- te = tempTe;
- card = null;
- }
- }
-
- @Override
- public int hashCode() {
- int result = 7;
- if (card != null) {
- result = result * 17 + card.hashCode();
- } else {
- result = result * 17 + te.hashCode();
- }
- return result;
-
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof TripleCard) {
- TripleCard comp = (TripleCard) o;
- if (card != null) {
- return card.equals(comp.card);
- } else {
- return te.equals(comp.te);
- }
- }
- return false;
- }
-
- @Override
- public String toString() {
- if (card != null) {
- return card.toString();
- } else {
- return te.toString();
- }
- }
-
- @Override
- public int compareTo(TripleCard o) {
-
- if (card != null) {
- return card.compareTo(o.card);
- } else {
- return te.compareTo(o.te);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleEntry.java
----------------------------------------------------------------------
diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleEntry.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleEntry.java
deleted file mode 100644
index 7deb346..0000000
--- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleEntry.java
+++ /dev/null
@@ -1,180 +0,0 @@
-package mvm.rya.joinselect.mr.utils;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
-
-public class TripleEntry implements WritableComparable<TripleEntry> {
-
- private Text first;
- private Text second;
- private Text firstPos;
- private Text secondPos;
- private Text keyPos;
-
- public TripleEntry() {
-
- first = new Text();
- second = new Text();
- firstPos = new Text();
- secondPos = new Text();
- keyPos = new Text();
-
- }
-
- public TripleEntry(String first, String second, String firstPos, String secondPos, String keyPos) {
- this.first = new Text(first);
- this.second = new Text(second);
- this.firstPos = new Text(firstPos);
- this.secondPos = new Text(secondPos);
- this.keyPos = new Text(keyPos);
- }
-
- public TripleEntry(Text first, Text second, Text firstPos, Text secondPos, Text keyPos) {
- this.first = first;
- this.second = second;
- this.firstPos = firstPos;
- this.secondPos = secondPos;
- this.keyPos = keyPos;
- }
-
- public void setEntry(Text first, Text second) {
- this.first = first;
- this.second = second;
- }
-
- public void setPosition(Text firstPos, Text secondPos, Text keyPos) {
- this.firstPos = firstPos;
- this.secondPos = secondPos;
- this.keyPos = keyPos;
- }
-
- public void setTE(TripleEntry te) {
-
- this.first.set(te.first);
- this.second.set(te.second);
- this.firstPos.set(te.firstPos);
- this.secondPos.set(te.secondPos);
- this.keyPos.set(te.keyPos);
-
- }
-
- public Text getFirst() {
- return this.first;
- }
-
- public Text getSecond() {
- return this.second;
- }
-
- public Text getFirstPos() {
- return this.firstPos;
- }
-
- public Text getSecondPos() {
- return this.secondPos;
- }
-
- public Text getKeyPos() {
- return this.keyPos;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- first.write(out);
- second.write(out);
- firstPos.write(out);
- secondPos.write(out);
- keyPos.write(out);
-
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- first.readFields(in);
- second.readFields(in);
- firstPos.readFields(in);
- secondPos.readFields(in);
- keyPos.readFields(in);
-
- }
-
- @Override
- public int hashCode() {
- int result = 7;
- result = result * 17 + first.hashCode();
- result = result * 17 + second.hashCode();
- result = result * 17 + firstPos.hashCode();
- result = result * 17 + secondPos.hashCode();
- result = result * 17 + keyPos.hashCode();
-
- return result;
-
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof TripleEntry) {
- TripleEntry trip = (TripleEntry) o;
- return first.equals(trip.first) && second.equals(trip.second) && firstPos.equals(trip.firstPos) && secondPos.equals(trip.secondPos)
- && keyPos.equals(trip.keyPos);
-
- }
- return false;
- }
-
- @Override
- public String toString() {
- return first + "\t" + firstPos + "\t" + second + "\t" + secondPos + "\t" + keyPos;
-
- }
-
- @Override
- public int compareTo(TripleEntry o) {
-
- int cmp = first.compareTo(o.first);
- if (cmp != 0) {
- return cmp;
- }
- cmp = firstPos.compareTo(o.firstPos);
- if (cmp != 0) {
- return cmp;
- }
- cmp = second.compareTo(o.second);
- if (cmp != 0) {
- return cmp;
- }
-
- cmp = secondPos.compareTo(o.secondPos);
- if (cmp != 0) {
- return cmp;
- }
- return keyPos.compareTo(o.keyPos);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/resources/META-INF/services/mvm.rya.prospector.plans.IndexWorkPlan
----------------------------------------------------------------------
diff --git a/extras/rya.prospector/src/main/resources/META-INF/services/mvm.rya.prospector.plans.IndexWorkPlan b/extras/rya.prospector/src/main/resources/META-INF/services/mvm.rya.prospector.plans.IndexWorkPlan
deleted file mode 100644
index 38258c1..0000000
--- a/extras/rya.prospector/src/main/resources/META-INF/services/mvm.rya.prospector.plans.IndexWorkPlan
+++ /dev/null
@@ -1 +0,0 @@
-mvm.rya.prospector.plans.impl.CountPlan
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/mr/ProspectorTest.groovy
----------------------------------------------------------------------
diff --git a/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/mr/ProspectorTest.groovy b/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/mr/ProspectorTest.groovy
deleted file mode 100644
index 766a239..0000000
--- a/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/mr/ProspectorTest.groovy
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package mvm.rya.prospector.mr
-
-import com.google.common.collect.Iterators
-import com.google.common.collect.Lists
-import mvm.rya.accumulo.AccumuloRyaDAO
-import mvm.rya.accumulo.AccumuloRdfConfiguration
-import mvm.rya.api.persist.RdfEvalStatsDAO
-import mvm.rya.api.domain.RyaStatement
-import mvm.rya.api.domain.RyaType
-import mvm.rya.api.domain.RyaURI
-import mvm.rya.prospector.domain.IndexEntry
-import mvm.rya.prospector.domain.TripleValueType
-import mvm.rya.prospector.service.ProspectorService
-import mvm.rya.prospector.service.ProspectorServiceEvalStatsDAO
-import mvm.rya.prospector.utils.ProspectorConstants
-import org.apache.accumulo.core.client.Instance
-import org.apache.accumulo.core.client.mock.MockInstance
-import org.apache.accumulo.core.security.Authorizations
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.util.ToolRunner
-import org.junit.Test
-import org.openrdf.model.vocabulary.XMLSchema
-import org.openrdf.model.impl.URIImpl
-
-import static org.junit.Assert.assertEquals
-import org.openrdf.model.impl.LiteralImpl
-import org.openrdf.model.Value
-
-/**
- * Date: 12/4/12
- * Time: 4:33 PM
- */
-class ProspectorTest {
-
- @Test
- public void testCount() throws Exception {
-
- Instance mock = new MockInstance("accumulo");
-
- def connector = mock.getConnector("user", "pass".bytes)
- def intable = "rya_spo"
- def outtable = "rya_prospects"
- if (connector.tableOperations().exists(outtable))
- connector.tableOperations().delete(outtable)
- connector.tableOperations().create(outtable)
-
- AccumuloRyaDAO ryaDAO = new AccumuloRyaDAO();
- ryaDAO.setConnector(connector);
- ryaDAO.init()
-
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata1")))
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata2")))
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("12")))
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred"), new RyaType(XMLSchema.INTEGER, "12")))
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred1"), new RyaType("12")))
-
- def confFile = "stats_cluster_config.xml"
- def confPath = new Path(getClass().getClassLoader().getResource(confFile).toString())
- def args = (String[]) [confPath];
- ToolRunner.run(new Prospector(), args);
- debugTable(connector, outtable)
-
- def scanner = connector.createScanner(outtable, new Authorizations("U", "FOUO"))
- def iter = scanner.iterator()
-// assertEquals(11, Iterators.size(iter))
-
- ryaDAO.destroy()
-
- def conf = new Configuration()
- conf.addResource(confPath)
- // debugTable(mrInfo, outtable)
-
- def service = new ProspectorService(connector, outtable)
- def auths = (String[]) ["U", "FOUO"]
- def prospects = service.getProspects(auths)
- def plist = Lists.newArrayList(prospects)
- assertEquals(1, plist.size())
-
- def rdfConf = new AccumuloRdfConfiguration(conf)
- rdfConf.setAuths("U","FOUO")
-
- prospects = service.getProspectsInRange(System.currentTimeMillis() - 100000, System.currentTimeMillis() + 10000, auths)
- plist = Lists.newArrayList(prospects)
- assertEquals(1, plist.size())
-
- List<String> queryTerms = new ArrayList<String>();
- queryTerms.add("urn:gem:etype");
- def query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.entity.name(), queryTerms, XMLSchema.ANYURI.stringValue(), auths)
- assertEquals(1, query.size())
-// assertEquals(
-// new IndexEntry(index: ProspectorConstants.COUNT, data: "urn:gem:etype", dataType: XMLSchema.ANYURI.stringValue(),
-// tripleValueType: TripleValueType.entity, visibility: "", count: -1, timestamp: plist.get(0)),
-// query.get(0))
-
- queryTerms = new ArrayList<String>();
- queryTerms.add("urn:gem:etype#1234");
- query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.subject.name(), queryTerms, XMLSchema.ANYURI.stringValue(), auths)
- assertEquals(1, query.size())
-
- queryTerms = new ArrayList<String>();
- queryTerms.add("urn:gem#pred");
- query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.predicate.name(), queryTerms, XMLSchema.ANYURI.stringValue(), auths)
- assertEquals(1, query.size())
- assertEquals(
- new IndexEntry(index: ProspectorConstants.COUNT, data: "urn:gem#pred", dataType: XMLSchema.ANYURI.stringValue(),
- tripleValueType: TripleValueType.predicate, visibility: "", count: 4l, timestamp: plist.get(0)),
- query.get(0))
-
- queryTerms = new ArrayList<String>();
- queryTerms.add("mydata1");
- query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.object.name(), queryTerms, XMLSchema.STRING.stringValue(), auths)
- assertEquals(1, query.size())
-// assertEquals(
-// new IndexEntry(index: ProspectorConstants.COUNT, data: "mydata1", dataType: XMLSchema.STRING.stringValue(),
-// tripleValueType: TripleValueType.object, visibility: "", count: -1, timestamp: plist.get(0)),
-// query.get(0))
-
- queryTerms = new ArrayList<String>();
- queryTerms.add("urn:gem:etype#1234");
- queryTerms.add("urn:gem#pred");
- query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.subjectpredicate.name(), queryTerms, XMLSchema.STRING.stringValue(), auths)
- assertEquals(1, query.size())
-// assertEquals(
-// new IndexEntry(index: ProspectorConstants.COUNT, data: "urn:gem:etype#1234" + "\u0000" + "urn:gem#pred", dataType: XMLSchema.STRING.stringValue(),
-// tripleValueType: TripleValueType.subjectpredicate, visibility: "", count: -1, timestamp: plist.get(0)),
-// query.get(0))
-
- queryTerms = new ArrayList<String>();
- queryTerms.add("urn:gem#pred");
- queryTerms.add("12");
- query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.predicateobject.name(), queryTerms, XMLSchema.STRING.stringValue(), auths)
- assertEquals(1, query.size())
-// assertEquals(
-// new IndexEntry(index: ProspectorConstants.COUNT, data: "urn:gem#pred" + "\u0000" + "12", dataType: XMLSchema.STRING.stringValue(),
-// tripleValueType: TripleValueType.predicateobject, visibility: "", count: -1, timestamp: plist.get(0)),
-// query.get(0))
-
- queryTerms = new ArrayList<String>();
- queryTerms.add("urn:gem:etype#1234");
- queryTerms.add("mydata1");
- query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.subjectobject.name(), queryTerms, XMLSchema.STRING.stringValue(), auths)
-
- assertEquals(1, query.size())
-// assertEquals(
-// new IndexEntry(index: ProspectorConstants.COUNT, data: "urn:gem:etype#1234" + "\u0000" + "mydata1", dataType: XMLSchema.STRING.stringValue(),
-// tripleValueType: TripleValueType.subjectobject, visibility: "", count: -1, timestamp: plist.get(0)),
-// query.get(0))
-
- //should be in a teardown method
- connector.tableOperations().delete(outtable)
- }
-
- private void debugTable(def connector, String table) {
- connector.createScanner(table, new Authorizations((String[]) ["U", "FOUO"])).iterator().each {
- println it
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/service/ProspectorServiceEvalStatsDAOTest.groovy
----------------------------------------------------------------------
diff --git a/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/service/ProspectorServiceEvalStatsDAOTest.groovy b/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/service/ProspectorServiceEvalStatsDAOTest.groovy
deleted file mode 100644
index 5bbbee8..0000000
--- a/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/service/ProspectorServiceEvalStatsDAOTest.groovy
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package mvm.rya.prospector.service
-
-import com.google.common.collect.Iterators
-import mvm.rya.accumulo.AccumuloRdfConfiguration
-import mvm.rya.accumulo.AccumuloRyaDAO
-import mvm.rya.api.domain.RyaStatement
-import mvm.rya.api.domain.RyaType
-import mvm.rya.api.domain.RyaURI
-import mvm.rya.api.persist.RdfEvalStatsDAO
-import mvm.rya.prospector.mr.Prospector
-import org.apache.accumulo.core.client.Instance
-import org.apache.accumulo.core.client.mock.MockInstance
-import org.apache.accumulo.core.security.Authorizations
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.util.ToolRunner
-import org.junit.Test
-import org.openrdf.model.impl.URIImpl
-import org.openrdf.model.vocabulary.XMLSchema
-
-import static org.junit.Assert.assertEquals
-import org.openrdf.model.impl.LiteralImpl
-import org.openrdf.model.Value
-
-/**
- * Date: 1/26/13
- * Time: 3:00 PM
- */
-class ProspectorServiceEvalStatsDAOTest {
-
- @Test
- public void testCount() throws Exception {
-
- Instance mock = new MockInstance("accumulo");
-
- def connector = mock.getConnector("user", "pass".bytes)
- def intable = "rya_spo"
- def outtable = "rya_prospects"
- if (connector.tableOperations().exists(outtable))
- connector.tableOperations().delete(outtable)
- connector.tableOperations().create(outtable)
-
- AccumuloRyaDAO ryaDAO = new AccumuloRyaDAO();
- ryaDAO.setConnector(connector);
- ryaDAO.init()
-
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata1")))
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata2")))
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("12")))
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred"), new RyaType(XMLSchema.INTEGER, "12")))
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred1"), new RyaType("12")))
-
- def confFile = "stats_cluster_config.xml"
- def confPath = new Path(getClass().getClassLoader().getResource(confFile).toString())
- def args = (String[]) [confPath];
- ToolRunner.run(new Prospector(), args);
- debugTable(connector, outtable)
-
- def scanner = connector.createScanner(outtable, new Authorizations("U", "FOUO"))
- def iter = scanner.iterator()
-// assertEquals(11, Iterators.size(iter))
-
- ryaDAO.destroy()
-
- def conf = new Configuration()
- conf.addResource(confPath)
-// debugTable(connector, outtable)
-
- def rdfConf = new AccumuloRdfConfiguration(conf)
- rdfConf.setAuths("U","FOUO")
- def evalDao = new ProspectorServiceEvalStatsDAO(connector, rdfConf)
- evalDao.init()
-
- List<Value> values = new ArrayList<Value>();
- values.add( new URIImpl("urn:gem#pred"));
-
- def count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.PREDICATE, values)
- assertEquals(4.0, count, 0.001);
-
- values = new ArrayList<Value>();
- values.add( new LiteralImpl("mydata1"));
-
- count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values);
- assertEquals(1.0, count, 0.001);
-
- values = new ArrayList<Value>();
- values.add( new LiteralImpl("mydata3"));
-
- count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values);
- assertEquals(-1.0, count, 0.001);
-
- //should be in a teardown method
- connector.tableOperations().delete(outtable)
- }
-
- @Test
- public void testNoAuthsCount() throws Exception {
-
- Instance mock = new MockInstance("accumulo");
- def connector = mock.getConnector("user", "pass".bytes)
- def intable = "rya_spo"
- def outtable = "rya_prospects"
- if (connector.tableOperations().exists(outtable))
- connector.tableOperations().delete(outtable)
- connector.tableOperations().create(outtable)
- connector.securityOperations().createUser("user", "pass".bytes, new Authorizations("U", "FOUO"))
-
- AccumuloRyaDAO ryaDAO = new AccumuloRyaDAO();
- ryaDAO.setConnector(connector);
- ryaDAO.init()
-
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata1")))
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata2")))
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("12")))
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred"), new RyaType(XMLSchema.INTEGER, "12")))
- ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred1"), new RyaType("12")))
-
- def confFile = "stats_cluster_config.xml"
- def confPath = new Path(getClass().getClassLoader().getResource(confFile).toString())
- def args = (String[]) [confPath];
- ToolRunner.run(new Prospector(), args);
-
- def scanner = connector.createScanner(outtable, new Authorizations("U", "FOUO"))
- def iter = scanner.iterator()
-// assertEquals(11, Iterators.size(iter))
-
- ryaDAO.destroy()
-
- def conf = new Configuration()
- conf.addResource(confPath)
-
- def rdfConf = new AccumuloRdfConfiguration(conf)
-// rdfConf.setAuths("U","FOUO")
- def evalDao = new ProspectorServiceEvalStatsDAO(connector, rdfConf)
- evalDao.init()
-
-
- List<Value> values = new ArrayList<Value>();
- values.add( new URIImpl("urn:gem#pred"));
- def count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.PREDICATE, values)
- assertEquals(4.0, count, 0.001);
-
- values = new ArrayList<Value>();
- values.add( new LiteralImpl("mydata1"));
- count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values);
- assertEquals(1.0, count, 0.001);
-
- values = new ArrayList<Value>();
- values.add( new LiteralImpl("mydata3"));
-
- count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values);
- assertEquals(-1.0, count, 0.001);
-
- //should be in a teardown method
- connector.tableOperations().delete(outtable)
- }
-
- private void debugTable(def connector, String table) {
- connector.createScanner(table, new Authorizations((String[]) ["U", "FOUO"])).iterator().each {
- println it
- }
- }
-}