You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pirk.apache.org by ea...@apache.org on 2016/08/11 13:19:34 UTC
incubator-pirk git commit: [PIRK-47] Change Query name and number to
a unique identifier - closes apache/incubator-pirk#54
Repository: incubator-pirk
Updated Branches:
refs/heads/master 10e08c434 -> 6c1e43efa
[PIRK-47] Change Query name and number to a unique identifier - closes apache/incubator-pirk#54
Project: http://git-wip-us.apache.org/repos/asf/incubator-pirk/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-pirk/commit/6c1e43ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-pirk/tree/6c1e43ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-pirk/diff/6c1e43ef
Branch: refs/heads/master
Commit: 6c1e43efacee7a6dd0152c41cd6e02dcb2e15e4b
Parents: 10e08c4
Author: tellison <te...@apache.org>
Authored: Thu Aug 11 09:19:13 2016 -0400
Committer: eawilliams <ea...@apache.org>
Committed: Thu Aug 11 09:19:13 2016 -0400
----------------------------------------------------------------------
.../org/apache/pirk/encryption/Paillier.java | 2 -
.../apache/pirk/querier/wideskies/Querier.java | 17 +----
.../pirk/querier/wideskies/QuerierDriver.java | 13 ++--
.../querier/wideskies/QuerierDriverCLI.java | 7 ---
.../pirk/querier/wideskies/QuerierProps.java | 11 +---
.../wideskies/decrypt/DecryptResponse.java | 2 +-
.../querier/wideskies/encrypt/EncryptQuery.java | 2 +-
.../apache/pirk/query/wideskies/QueryInfo.java | 65 ++++++++------------
.../mapreduce/ComputeResponseTool.java | 4 +-
.../wideskies/spark/ComputeExpLookupTable.java | 2 +-
.../pirk/schema/response/QueryResponseJSON.java | 6 +-
.../distributed/testsuite/DistTestSuite.java | 39 +++---------
.../org/apache/pirk/test/utils/BaseTests.java | 33 +++++-----
.../apache/pirk/test/utils/StandaloneQuery.java | 10 +--
.../org/apache/pirk/test/utils/TestUtils.java | 9 +--
src/main/resources/querier.properties | 3 -
16 files changed, 75 insertions(+), 150 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/encryption/Paillier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/encryption/Paillier.java b/src/main/java/org/apache/pirk/encryption/Paillier.java
index dec6c30..6c62fe8 100644
--- a/src/main/java/org/apache/pirk/encryption/Paillier.java
+++ b/src/main/java/org/apache/pirk/encryption/Paillier.java
@@ -144,8 +144,6 @@ public class Paillier implements Cloneable, Serializable
public Paillier(int bitLengthInput, int certainty) throws PIRException
{
this(bitLengthInput, certainty, -1);
-
- logger.info("Parameters = " + parametersToString());
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/querier/wideskies/Querier.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/querier/wideskies/Querier.java b/src/main/java/org/apache/pirk/querier/wideskies/Querier.java
index 4d6523d..0a94541 100644
--- a/src/main/java/org/apache/pirk/querier/wideskies/Querier.java
+++ b/src/main/java/org/apache/pirk/querier/wideskies/Querier.java
@@ -24,10 +24,7 @@ import java.util.HashMap;
import org.apache.pirk.encryption.Paillier;
import org.apache.pirk.query.wideskies.Query;
-import org.apache.pirk.query.wideskies.QueryInfo;
import org.apache.pirk.serialization.Storable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Class to hold the information necessary for the PIR querier to perform decryption
@@ -37,10 +34,6 @@ public class Querier implements Serializable, Storable
{
private static final long serialVersionUID = 1L;
- private static final Logger logger = LoggerFactory.getLogger(Querier.class);
-
- private QueryInfo queryInfo = null;
-
private Query query = null; // contains the query vectors and functionality
private Paillier paillier = null; // Paillier encryption functionality
@@ -52,11 +45,8 @@ public class Querier implements Serializable, Storable
// if the selector is of variable lengths
private HashMap<Integer,String> embedSelectorMap = null;
- public Querier(QueryInfo queryInfoInput, ArrayList<String> selectorsInput, Paillier paillierInput, Query pirQueryInput,
- HashMap<Integer,String> embedSelectorMapInput)
+ public Querier(ArrayList<String> selectorsInput, Paillier paillierInput, Query pirQueryInput, HashMap<Integer,String> embedSelectorMapInput)
{
- queryInfo = queryInfoInput;
-
selectors = selectorsInput;
paillier = paillierInput;
@@ -66,11 +56,6 @@ public class Querier implements Serializable, Storable
embedSelectorMap = embedSelectorMapInput;
}
- public QueryInfo getPirWatchlist()
- {
- return queryInfo;
- }
-
public Query getPirQuery()
{
return query;
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java
index c8647c6..6061f3a 100644
--- a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java
+++ b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.ArrayList;
+import java.util.UUID;
import org.apache.pirk.encryption.Paillier;
import org.apache.pirk.querier.wideskies.decrypt.DecryptResponse;
@@ -92,7 +93,6 @@ public class QuerierDriver implements Serializable
int dataPartitionBitSize = 0;
int paillierBitSize = 0;
int certainty = 0;
- String queryName = null;
int bitSet = -1;
boolean embedSelector = true;
boolean useMemLookupTable = false;
@@ -112,7 +112,6 @@ public class QuerierDriver implements Serializable
if (action.equals("encrypt"))
{
queryType = SystemConfiguration.getProperty(QuerierProps.QUERYTYPE);
- queryName = SystemConfiguration.getProperty(QuerierProps.QUERYNAME);
hashBitSize = Integer.parseInt(SystemConfiguration.getProperty(QuerierProps.HASHBITSIZE));
hashKey = SystemConfiguration.getProperty(QuerierProps.HASHBITSIZE);
dataPartitionBitSize = Integer.parseInt(SystemConfiguration.getProperty(QuerierProps.DATAPARTITIONSIZE));
@@ -157,17 +156,17 @@ public class QuerierDriver implements Serializable
+ "\n hashBitSize = " + hashBitSize + "\n hashKey = " + hashKey + "\n dataPartitionBitSize = " + dataPartitionBitSize + "\n paillierBitSize = "
+ paillierBitSize + "\n certainty = " + certainty);
- // Read in the selectors and extract the queryNum - first line in the file
+ // Read in the selectors and extract the queryIdentifier - first line in the file
ArrayList<String> selectors = FileIOUtils.readToArrayList(inputFile);
- double queryNum = Double.parseDouble(selectors.get(0));
+ UUID queryIdentifier = UUID.fromString(selectors.get(0));
selectors.remove(0);
int numSelectors = selectors.size();
- logger.info("queryNum = " + queryNum + " numSelectors = " + numSelectors);
+ logger.info("queryIdentifier = " + queryIdentifier + " numSelectors = " + numSelectors);
// Set the necessary QueryInfo and Paillier objects
- QueryInfo queryInfo = new QueryInfo(queryNum, numSelectors, hashBitSize, hashKey, dataPartitionBitSize, queryType, queryName, paillierBitSize,
- useMemLookupTable, embedSelector, useHDFSLookupTable);
+ QueryInfo queryInfo = new QueryInfo(queryIdentifier, numSelectors, hashBitSize, hashKey, dataPartitionBitSize, queryType, useMemLookupTable,
+ embedSelector, useHDFSLookupTable);
if (SystemConfiguration.isSetTrue("pir.embedQuerySchema"))
{
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java
index a5ea321..2bd581c 100644
--- a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java
+++ b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java
@@ -211,13 +211,6 @@ public class QuerierDriverCLI
optionTYPE.setType(String.class);
options.addOption(optionTYPE);
- // NAME
- Option optionNAME = new Option("qn", QuerierProps.QUERYNAME, true, "required for encryption -- Name of the query");
- optionNAME.setRequired(false);
- optionNAME.setArgName(QuerierProps.QUERYNAME);
- optionNAME.setType(String.class);
- options.addOption(optionNAME);
-
// HASHBITSIZE
Option optionHASHBITSIZE = new Option("hb", QuerierProps.HASHBITSIZE, true, "required -- Bit size of keyed hash");
optionHASHBITSIZE.setRequired(false);
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/querier/wideskies/QuerierProps.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/querier/wideskies/QuerierProps.java b/src/main/java/org/apache/pirk/querier/wideskies/QuerierProps.java
index e8820af..38fb088 100644
--- a/src/main/java/org/apache/pirk/querier/wideskies/QuerierProps.java
+++ b/src/main/java/org/apache/pirk/querier/wideskies/QuerierProps.java
@@ -48,7 +48,6 @@ public class QuerierProps
public static final String PAILLIERBITSIZE = "querier.paillierBitSize";
public static final String BITSET = "querier.bitSet";
public static final String CERTAINTY = "querier.certainty";
- public static final String QUERYNAME = "querier.queryName";
public static final String QUERYSCHEMAS = "querier.querySchemas";
public static final String DATASCHEMAS = "querier.dataSchemas";
public static final String EMBEDSELECTOR = "querier.embedSelector";
@@ -62,8 +61,8 @@ public class QuerierProps
public static final String QUERIERFILE = "querier.querierFile";
public static final List<String> PROPSLIST = Arrays.asList(ACTION, INPUTFILE, OUTPUTFILE, QUERYTYPE, NUMTHREADS, EMBEDQUERYSCHEMA, HASHBITSIZE, HASHKEY,
- DATAPARTITIONSIZE, PAILLIERBITSIZE, BITSET, CERTAINTY, QUERYNAME, QUERYSCHEMAS, DATASCHEMAS, EMBEDSELECTOR, USEMEMLOOKUPTABLE, USEHDFSLOOKUPTABLE,
- SR_ALGORITHM, SR_PROVIDER);
+ DATAPARTITIONSIZE, PAILLIERBITSIZE, BITSET, CERTAINTY, QUERYSCHEMAS, DATASCHEMAS, EMBEDSELECTOR, USEMEMLOOKUPTABLE, USEHDFSLOOKUPTABLE, SR_ALGORITHM,
+ SR_PROVIDER);
/**
* Validates the querier properties
@@ -151,12 +150,6 @@ public class QuerierProps
valid = false;
}
- if (!SystemConfiguration.hasProperty(QUERYNAME))
- {
- logger.info("For action='encrypt': Must have the option " + QUERYNAME);
- valid = false;
- }
-
if (!SystemConfiguration.hasProperty(BITSET))
{
logger.info("For action='encrypt': Must have the option " + BITSET);
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java b/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java
index 8eebf82..a9004fe 100644
--- a/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java
+++ b/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java
@@ -139,7 +139,7 @@ public class DecryptResponse
// Create the runnable and execute
// selectorMaskMap and rElements are synchronized, pirWatchlist is copied, selectors is partitioned
- DecryptResponseRunnable runDec = new DecryptResponseRunnable(rElements, selectorsPartition, selectorMaskMap, queryInfo.copy(), embedSelectorMap);
+ DecryptResponseRunnable runDec = new DecryptResponseRunnable(rElements, selectorsPartition, selectorMaskMap, queryInfo.clone(), embedSelectorMap);
runnables.add(runDec);
es.execute(runDec);
}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java b/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java
index 9e2099a..e02dacc 100644
--- a/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java
+++ b/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java
@@ -158,7 +158,7 @@ public class EncryptQuery
}
// Set the Querier object
- querier = new Querier(queryInfo, selectors, paillier, query, embedSelectorMap);
+ querier = new Querier(selectors, paillier, query, embedSelectorMap);
}
private HashMap<Integer,Integer> computeSelectorQueryVecMap()
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java b/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java
index 8c69fa9..6a3bb15 100644
--- a/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java
+++ b/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java
@@ -19,6 +19,7 @@
package org.apache.pirk.query.wideskies;
import java.io.Serializable;
+import java.util.UUID;
import org.apache.pirk.schema.query.QuerySchema;
import org.apache.pirk.schema.query.QuerySchemaRegistry;
@@ -32,21 +33,17 @@ import org.slf4j.LoggerFactory;
* we no longer have collisions
*
*/
-public class QueryInfo implements Serializable
+public class QueryInfo implements Serializable, Cloneable
{
private static final long serialVersionUID = 1L;
private static final Logger logger = LoggerFactory.getLogger(QueryInfo.class);
- private double queryNum = 0.0; // the identifier num of the query
+ private UUID identifier; // the identifier of the query
private int numSelectors = 0; // the number of selectors in the query, given by \floor{paillerBitSize/dataPartitionBitSize}
private String queryType = null; // QueryType string const
- private String queryName = null; // Name of query
-
- private int paillierBitSize = 0; // Paillier modulus size
-
private int hashBitSize = 0; // Bit size of the keyed hash function
private String hashKey = null; // Key for the keyed hash function
@@ -66,13 +63,18 @@ public class QueryInfo implements Serializable
QuerySchema qSchema = null;
- public QueryInfo(double queryNumInput, int numSelectorsInput, int hashBitSizeInput, String hashKeyInput, int dataPartitionBitSizeInput,
- String queryTypeInput, String queryNameInput, int paillierBitSizeIn, boolean useExpLookupTableInput, boolean embedSelectorInput,
- boolean useHDFSExpLookupTableInput)
+ public QueryInfo(int numSelectorsInput, int hashBitSizeInput, String hashKeyInput, int dataPartitionBitSizeInput, String queryTypeInput,
+ boolean useExpLookupTableInput, boolean embedSelectorInput, boolean useHDFSExpLookupTableInput)
+ {
+ this(UUID.randomUUID(), numSelectorsInput, hashBitSizeInput, hashKeyInput, dataPartitionBitSizeInput, queryTypeInput, useExpLookupTableInput,
+ embedSelectorInput, useHDFSExpLookupTableInput);
+ }
+
+ public QueryInfo(UUID identifierInput, int numSelectorsInput, int hashBitSizeInput, String hashKeyInput, int dataPartitionBitSizeInput,
+ String queryTypeInput, boolean useExpLookupTableInput, boolean embedSelectorInput, boolean useHDFSExpLookupTableInput)
{
- queryNum = queryNumInput;
+ identifier = identifierInput;
queryType = queryTypeInput;
- queryName = queryNameInput;
numSelectors = numSelectorsInput;
@@ -87,8 +89,6 @@ public class QueryInfo implements Serializable
dataPartitionBitSize = dataPartitionBitSizeInput;
numPartitionsPerDataElement = numBitsPerDataElement / dataPartitionBitSizeInput;
- paillierBitSize = paillierBitSizeIn;
-
if (embedSelectorInput)
{
numPartitionsPerDataElement += 4; // using a 8-bit partition size and a 32-bit embedded selector
@@ -97,16 +97,9 @@ public class QueryInfo implements Serializable
printQueryInfo();
}
- public QueryInfo(double queryNumInput, int numSelectorsInput, int hashBitSizeInput, String hashKeyInput, int dataPartitionBitSizeInput,
- String queryTypeInput, String queryNameInput, int paillierBitSizeIn)
- {
- this(queryNumInput, numSelectorsInput, hashBitSizeInput, hashKeyInput, dataPartitionBitSizeInput, queryTypeInput, queryNameInput, paillierBitSizeIn, false,
- true, false);
- }
-
- public double getQueryNum()
+ public UUID getIdentifier()
{
- return queryNum;
+ return identifier;
}
public String getQueryType()
@@ -114,21 +107,11 @@ public class QueryInfo implements Serializable
return queryType;
}
- public String getQueryName()
- {
- return queryName;
- }
-
public int getNumSelectors()
{
return numSelectors;
}
- public int getPaillierBitSize()
- {
- return paillierBitSize;
- }
-
public int getHashBitSize()
{
return hashBitSize;
@@ -181,15 +164,21 @@ public class QueryInfo implements Serializable
public void printQueryInfo()
{
- logger.info("queryNum = " + queryNum + " numSelectors = " + numSelectors + " hashBitSize = " + hashBitSize + " hashKey = " + hashKey
+ logger.info("identifier = " + identifier + " numSelectors = " + numSelectors + " hashBitSize = " + hashBitSize + " hashKey = " + hashKey
+ " dataPartitionBitSize = " + dataPartitionBitSize + " numBitsPerDataElement = " + numBitsPerDataElement + " numPartitionsPerDataElement = "
- + numPartitionsPerDataElement + " queryType = " + queryType + " queryName = " + queryName + " paillierBitSize = " + paillierBitSize
- + " useExpLookupTable = " + useExpLookupTable + " useHDFSExpLookupTable = " + useHDFSExpLookupTable + " embedSelector = " + embedSelector);
+ + numPartitionsPerDataElement + " queryType = " + queryType + " useExpLookupTable = " + useExpLookupTable + " useHDFSExpLookupTable = "
+ + useHDFSExpLookupTable + " embedSelector = " + embedSelector);
}
- public QueryInfo copy()
+ @Override
+ public QueryInfo clone()
{
- return new QueryInfo(this.queryNum, this.numSelectors, this.hashBitSize, this.hashKey, this.dataPartitionBitSize, this.queryType, this.queryName,
- this.paillierBitSize, this.useExpLookupTable, this.embedSelector, this.useHDFSExpLookupTable);
+ try
+ {
+ return (QueryInfo) super.clone();
+ } catch (CloneNotSupportedException e)
+ {
+ throw new RuntimeException(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java b/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java
index 15d62df..98f58c6 100644
--- a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java
+++ b/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java
@@ -237,7 +237,7 @@ public class ComputeResponseTool extends Configured implements Tool
logger.info("Creating expTable");
// The split location for the interim calculations, delete upon completion
- Path splitDir = new Path("/tmp/splits-" + queryInfo.getQueryNum());
+ Path splitDir = new Path("/tmp/splits-" + queryInfo.getIdentifier());
if (fs.exists(splitDir))
{
fs.delete(splitDir, true);
@@ -263,7 +263,7 @@ public class ComputeResponseTool extends Configured implements Tool
// Run the job to generate the expTable
// Job jobExp = new Job(mrConfig.getConfig(), "pirExp-" + pirWL.getWatchlistNum());
- Job jobExp = new Job(conf, "pirExp-" + queryInfo.getQueryNum());
+ Job jobExp = new Job(conf, "pirExp-" + queryInfo.getIdentifier());
jobExp.setSpeculativeExecution(false);
jobExp.getConfiguration().set("mapreduce.map.speculative", "false");
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/responder/wideskies/spark/ComputeExpLookupTable.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/ComputeExpLookupTable.java b/src/main/java/org/apache/pirk/responder/wideskies/spark/ComputeExpLookupTable.java
index 969b6ec..6b64e35 100644
--- a/src/main/java/org/apache/pirk/responder/wideskies/spark/ComputeExpLookupTable.java
+++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/ComputeExpLookupTable.java
@@ -67,7 +67,7 @@ public class ComputeExpLookupTable
{
JavaPairRDD<Integer,Iterable<Tuple2<Integer,BigInteger>>> expCalculations;
- logger.info("Creating expTable in hdfs for queryName = " + query.getQueryInfo().getQueryName());
+ logger.info("Creating expTable in hdfs for query identifier = " + query.getQueryInfo().getIdentifier());
// Prep the output directory
Path outPathExp = new Path(outputDirExp);
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java b/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java
index e9c1161..d32d69b 100644
--- a/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java
+++ b/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java
@@ -57,9 +57,6 @@ public class QueryResponseJSON implements Serializable
public static final String QUERY_ID = "query_id"; // query ID that generated the notification
public static final Text QUERY_ID_TEXT = new Text(QUERY_ID);
- public static final String QUERY_NAME = "query_name"; // name of the query that generated the notification
- public static final Text QUERY_NAME_TEXT = new Text(QUERY_NAME);
-
public static final String SELECTOR = "match"; // tag for selector that generated the hit
public static final Text SELECTOR_TEXT = new Text(SELECTOR);
@@ -203,8 +200,7 @@ public class QueryResponseJSON implements Serializable
public void setGeneralQueryResponseFields(QueryInfo queryInfo)
{
jsonObj.put(EVENT_TYPE, queryInfo.getQueryType());
- jsonObj.put(QUERY_ID, queryInfo.getQueryNum());
- jsonObj.put(QUERY_NAME, queryInfo.getQueryName());
+ jsonObj.put(QUERY_ID, queryInfo.getIdentifier().toString());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java b/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java
index 2e3464e..0556ead 100644
--- a/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java
+++ b/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java
@@ -20,6 +20,7 @@ package org.apache.pirk.test.distributed.testsuite;
import java.io.File;
import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -321,52 +322,32 @@ public class DistTestSuite
}
// Base method to perform query
- @SuppressWarnings("unused")
- public static ArrayList<QueryResponseJSON> performQuery(String queryType, ArrayList<String> selectors, FileSystem fs, boolean isSpark, int numThreads)
+ public static List<QueryResponseJSON> performQuery(String queryType, ArrayList<String> selectors, FileSystem fs, boolean isSpark, int numThreads)
throws Exception
{
logger.info("performQuery: ");
String queryInputDir = SystemConfiguration.getProperty(DistributedTestDriver.PIR_QUERY_INPUT_DIR);
String outputFile = SystemConfiguration.getProperty(DistributedTestDriver.OUTPUT_DIRECTORY_PROPERTY);
- Path outputFilePath = new Path(outputFile);
- if (fs.exists(outputFilePath))
- {
- fs.delete(new Path(outputFile), true);
- }
+ fs.delete(new Path(outputFile), true); // Ensure old output does not exist.
+
SystemConfiguration.setProperty("pir.queryInput", queryInputDir);
SystemConfiguration.setProperty("pir.outputFile", outputFile);
SystemConfiguration.setProperty("pir.numReduceTasks", "1");
SystemConfiguration.setProperty("pir.stopListFile", SystemConfiguration.getProperty(DistributedTestDriver.PIR_STOPLIST_FILE));
- ArrayList<QueryResponseJSON> results;
-
// Create the temp result file
File fileFinalResults = File.createTempFile("finalResultsFile", ".txt");
fileFinalResults.deleteOnExit();
logger.info("fileFinalResults = " + fileFinalResults.getAbsolutePath());
- boolean embedSelector = false;
- if (SystemConfiguration.getBooleanProperty("pirTest.embedSelector", false))
- {
- embedSelector = true;
- }
-
- boolean useExpLookupTable = false;
- if (SystemConfiguration.getBooleanProperty("pirTest.useExpLookupTable", false))
- {
- useExpLookupTable = true;
- }
-
- boolean useHDFSExpLookupTable = false;
- if (SystemConfiguration.getBooleanProperty("pirTest.useHDFSExpLookupTable", false))
- {
- useHDFSExpLookupTable = true;
- }
+ boolean embedSelector = SystemConfiguration.getBooleanProperty("pirTest.embedSelector", false);
+ boolean useExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useExpLookupTable", false);
+ boolean useHDFSExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useHDFSExpLookupTable", false);
// Set the necessary objects
- QueryInfo queryInfo = new QueryInfo(BaseTests.queryNum, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize,
- queryType, queryType + "_" + BaseTests.queryNum, BaseTests.paillierBitSize, useExpLookupTable, embedSelector, useHDFSExpLookupTable);
+ QueryInfo queryInfo = new QueryInfo(BaseTests.queryIdentifier, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize,
+ queryType, useExpLookupTable, embedSelector, useHDFSExpLookupTable);
Paillier paillier = new Paillier(BaseTests.paillierBitSize, BaseTests.certainty);
@@ -463,7 +444,7 @@ public class DistTestSuite
// Read in results
logger.info("Reading in and checking results");
- results = TestUtils.readResultsFile(fileFinalResults);
+ List<QueryResponseJSON> results = TestUtils.readResultsFile(fileFinalResults);
// Reset data and query schema properties
SystemConfiguration.setProperty("data.schemas", dataSchemaProp);
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/test/utils/BaseTests.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/test/utils/BaseTests.java b/src/main/java/org/apache/pirk/test/utils/BaseTests.java
index afb3144..e99744c 100644
--- a/src/main/java/org/apache/pirk/test/utils/BaseTests.java
+++ b/src/main/java/org/apache/pirk/test/utils/BaseTests.java
@@ -23,7 +23,9 @@ import static org.junit.Assert.fail;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
+import java.util.UUID;
import org.apache.hadoop.fs.FileSystem;
import org.apache.pirk.query.wideskies.QueryUtils;
@@ -45,10 +47,10 @@ public class BaseTests
{
private static final Logger logger = LoggerFactory.getLogger(BaseTests.class);
- public static double queryNum = 1.0;
+ public static UUID queryIdentifier = UUID.randomUUID();
public static int dataPartitionBitSize = 8;
- // Selectors for domain and IP queries, queryNum is the first entry for file generation
+ // Selectors for domain and IP queries, queryIdentifier is the first entry for file generation
private static ArrayList<String> selectorsDomain = new ArrayList<>(Arrays.asList("s.t.u.net", "d.e.com", "r.r.r.r", "a.b.c.com", "something.else", "x.y.net"));
private static ArrayList<String> selectorsIP = new ArrayList<>(Arrays.asList("55.55.55.55", "5.6.7.8", "10.20.30.40", "13.14.15.16", "21.22.23.24"));
@@ -78,7 +80,7 @@ public class BaseTests
QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_HOSTNAME_QUERY);
int numExpectedResults = 6;
- ArrayList<QueryResponseJSON> results;
+ List<QueryResponseJSON> results;
if (isDistributed)
{
results = DistTestSuite.performQuery(Inputs.DNS_HOSTNAME_QUERY, selectorsDomain, fs, isSpark, numThreads);
@@ -155,8 +157,7 @@ public class BaseTests
if (addElement)
{
QueryResponseJSON wlJSON = new QueryResponseJSON();
- wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryNum);
- wlJSON.setMapping(QueryResponseJSON.QUERY_NAME, Inputs.DNS_HOSTNAME_QUERY + "_" + queryNum);
+ wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier.toString());
wlJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_HOSTNAME_QUERY);
wlJSON.setMapping(Inputs.DATE, dataMap.get(Inputs.DATE));
wlJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP));
@@ -201,7 +202,7 @@ public class BaseTests
logger.info("Running testDNSIPQuery(): ");
QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_IP_QUERY);
- ArrayList<QueryResponseJSON> results;
+ List<QueryResponseJSON> results;
if (isDistributed)
{
@@ -237,8 +238,7 @@ public class BaseTests
if (addElement)
{
QueryResponseJSON wlJSON = new QueryResponseJSON();
- wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryNum);
- wlJSON.setMapping(QueryResponseJSON.QUERY_NAME, Inputs.DNS_IP_QUERY + "_" + queryNum);
+ wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier);
wlJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_IP_QUERY);
wlJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP));
wlJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
@@ -276,7 +276,7 @@ public class BaseTests
logger.info("Running testDNSNXDOMAINQuery(): ");
QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_NXDOMAIN_QUERY);
- ArrayList<QueryResponseJSON> results;
+ List<QueryResponseJSON> results;
if (isDistributed)
{
@@ -302,8 +302,7 @@ public class BaseTests
if (dataMap.get(Inputs.RCODE).toString().equals("3"))
{
QueryResponseJSON wlJSON = new QueryResponseJSON();
- wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryNum);
- wlJSON.setMapping(QueryResponseJSON.QUERY_NAME, Inputs.DNS_NXDOMAIN_QUERY + "_" + queryNum);
+ wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier);
wlJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_NXDOMAIN_QUERY);
wlJSON.setMapping(Inputs.QNAME, dataMap.get(Inputs.QNAME)); // this gets re-embedded as the original selector after decryption
wlJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
@@ -340,7 +339,7 @@ public class BaseTests
logger.info("Running testSRCIPQuery(): ");
QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_SRCIP_QUERY);
- ArrayList<QueryResponseJSON> results;
+ List<QueryResponseJSON> results;
int removeTailElements = 0;
int numExpectedResults = 1;
@@ -376,8 +375,7 @@ public class BaseTests
{
// Form the correct result QueryResponseJSON object
QueryResponseJSON qrJSON = new QueryResponseJSON();
- qrJSON.setMapping(QueryResponseJSON.QUERY_ID, queryNum);
- qrJSON.setMapping(QueryResponseJSON.QUERY_NAME, Inputs.DNS_SRCIP_QUERY + "_" + queryNum);
+ qrJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier);
qrJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_SRCIP_QUERY);
qrJSON.setMapping(Inputs.QNAME, parseString(dataMap, Inputs.QNAME));
qrJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
@@ -414,7 +412,7 @@ public class BaseTests
logger.info("Running testSRCIPQueryNoFilter(): ");
QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_SRCIP_QUERY_NO_FILTER);
- ArrayList<QueryResponseJSON> results;
+ List<QueryResponseJSON> results;
int numExpectedResults = 3;
if (isDistributed)
@@ -447,8 +445,7 @@ public class BaseTests
{
// Form the correct result QueryResponseJSON object
QueryResponseJSON qrJSON = new QueryResponseJSON();
- qrJSON.setMapping(QueryResponseJSON.QUERY_ID, queryNum);
- qrJSON.setMapping(QueryResponseJSON.QUERY_NAME, Inputs.DNS_SRCIP_QUERY_NO_FILTER + "_" + queryNum);
+ qrJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier);
qrJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_SRCIP_QUERY_NO_FILTER);
qrJSON.setMapping(Inputs.QNAME, parseString(dataMap, Inputs.QNAME));
qrJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
@@ -636,7 +633,7 @@ public class BaseTests
return set;
}
- private static void printResultList(ArrayList<QueryResponseJSON> list)
+ private static void printResultList(List<QueryResponseJSON> list)
{
for (QueryResponseJSON obj : list)
{
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java b/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java
index 4e98f5d..a4bf67d 100644
--- a/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java
+++ b/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java
@@ -24,6 +24,7 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import org.apache.pirk.encryption.Paillier;
import org.apache.pirk.querier.wideskies.Querier;
@@ -56,12 +57,11 @@ public class StandaloneQuery
String testQuerySchemaName = "testQuerySchema";
// Base method to perform the query
- public static ArrayList<QueryResponseJSON> performStandaloneQuery(ArrayList<JSONObject> dataElements, String queryType, ArrayList<String> selectors,
+ public static List<QueryResponseJSON> performStandaloneQuery(ArrayList<JSONObject> dataElements, String queryType, ArrayList<String> selectors,
int numThreads, boolean testFalsePositive) throws IOException, InterruptedException, PIRException
{
logger.info("Performing watchlisting: ");
- ArrayList<QueryResponseJSON> results = null;
QuerySchema qSchema = QuerySchemaRegistry.get(queryType);
// Create the necessary files
@@ -82,8 +82,8 @@ public class StandaloneQuery
boolean useHDFSExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useHDFSExpLookupTable", false);
// Set the necessary objects
- QueryInfo queryInfo = new QueryInfo(BaseTests.queryNum, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize,
- queryType, queryType + "_" + BaseTests.queryNum, BaseTests.paillierBitSize, useExpLookupTable, embedSelector, useHDFSExpLookupTable);
+ QueryInfo queryInfo = new QueryInfo(BaseTests.queryIdentifier, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize,
+ queryType, useExpLookupTable, embedSelector, useHDFSExpLookupTable);
if (SystemConfiguration.getBooleanProperty("pir.embedQuerySchema", false))
{
@@ -152,7 +152,7 @@ public class StandaloneQuery
// Read in results
logger.info("Reading in and checking results");
- results = TestUtils.readResultsFile(fileFinalResults);
+ List<QueryResponseJSON> results = TestUtils.readResultsFile(fileFinalResults);
// Clean up
fileQuerier.delete();
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/test/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/test/utils/TestUtils.java b/src/main/java/org/apache/pirk/test/utils/TestUtils.java
index e1e03dc..c30e204 100644
--- a/src/main/java/org/apache/pirk/test/utils/TestUtils.java
+++ b/src/main/java/org/apache/pirk/test/utils/TestUtils.java
@@ -267,14 +267,11 @@ public class TestUtils
/**
* Converts the result file into an ArrayList of QueryResponseJSON objects
*/
- public static ArrayList<QueryResponseJSON> readResultsFile(File file)
+ public static List<QueryResponseJSON> readResultsFile(File file)
{
- ArrayList<QueryResponseJSON> results = new ArrayList<>();
- try
+ List<QueryResponseJSON> results = new ArrayList<>();
+ try (BufferedReader br = new BufferedReader(new FileReader(file)))
{
- FileReader fr = new FileReader(file);
- BufferedReader br = new BufferedReader(fr);
-
String line;
while ((line = br.readLine()) != null)
{
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/resources/querier.properties
----------------------------------------------------------------------
diff --git a/src/main/resources/querier.properties b/src/main/resources/querier.properties
index 2051d46..88352d7 100644
--- a/src/main/resources/querier.properties
+++ b/src/main/resources/querier.properties
@@ -100,9 +100,6 @@ querier.numThreads=
#paillierBitSize -- required for encryption -- Paillier modulus size N
#querier.paillierBitSize=
-#queryName -- required for encryption -- Name of the query
-#querier.queryName=
-
#queryType -- required for encryption
#Type of the query as defined in the 'schemaName' tag of the corresponding query schema file
#querier.queryType=