You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pirk.apache.org by sm...@apache.org on 2016/08/18 15:31:52 UTC
[1/6] incubator-pirk git commit: Move all tests under src/test
directory, this closes apache/incubator-pirk#66
Repository: incubator-pirk
Updated Branches:
refs/heads/master 8a1c4e153 -> a643ae689
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/wideskies/standalone/StandaloneTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/wideskies/standalone/StandaloneTest.java b/src/test/java/org/apache/pirk/test/wideskies/standalone/StandaloneTest.java
new file mode 100644
index 0000000..5175470
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/wideskies/standalone/StandaloneTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.wideskies.standalone;
+
+import java.util.ArrayList;
+
+import org.apache.pirk.schema.data.DataSchemaRegistry;
+import org.apache.pirk.schema.query.QuerySchemaRegistry;
+import org.apache.pirk.schema.query.filter.StopListFilter;
+import org.apache.pirk.test.utils.BaseTests;
+import org.apache.pirk.test.utils.Inputs;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.json.simple.JSONObject;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Functional test suite for stand alone testing - non Spark applications
+ * <p>
+ * Tests low side module and basic encryption, decryption mechanisms
+ * <p>
+ * Using a fixed 8-bit data partition size (consistent with the currently codebase)
+ * <p>
+ * Runs with useExpLookupTable = false as generating the lookup table takes too long for normal in-memory builds
+ *
+ */
+public class StandaloneTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(StandaloneTest.class);
+
+ private static final String STOPLIST_FILE = "testStopListFile";
+
+ private static String stopListFileProp = null;
+
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ // Reset the schema properties and registries
+ DataSchemaRegistry.clearRegistry();
+ QuerySchemaRegistry.clearRegistry();
+ SystemConfiguration.setProperty("data.schemas", "none");
+ SystemConfiguration.setProperty("query.schemas", "none");
+
+ // Create the stoplist file
+ stopListFileProp = SystemConfiguration.getProperty("pir.stopListFile");
+ SystemConfiguration.setProperty("pir.stopListFile", STOPLIST_FILE);
+ String newSLFile = Inputs.createPIRStopList(null, false);
+ SystemConfiguration.setProperty("pir.stopListFile", newSLFile);
+ logger.info("stopListFileProp = " + stopListFileProp + " new prop = " + SystemConfiguration.getProperty("pir.stopListFile"));
+
+ // Create data and query schemas
+ Inputs.createSchemaFiles(StopListFilter.class.getName());
+ }
+
+ @AfterClass
+ public static void teardown()
+ {
+ // Reset the schema properties and registries
+ DataSchemaRegistry.clearRegistry();
+ QuerySchemaRegistry.clearRegistry();
+ SystemConfiguration.setProperty("data.schemas", "none");
+ SystemConfiguration.setProperty("query.schemas", "none");
+ }
+
+ @Test
+ public void runTests() throws Exception
+ {
+ ArrayList<JSONObject> dataElements = Inputs.createJSONDataElements();
+ ArrayList<JSONObject> dataElementsRcode3 = Inputs.getRcode3JSONDataElements();
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+ // Run tests and use the embedded selector
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, 1, false);
+ BaseTests.testSRCIPQuery(dataElements, 2);
+ BaseTests.testDNSIPQuery(dataElements, 3); // numThreads % num elements to encrypt != 0
+ BaseTests.testDNSNXDOMAINQuery(dataElementsRcode3, 4); // numThreads % num elements to encrypt = 0
+
+ // Test embedded QuerySchema
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+ BaseTests.testDNSHostnameQuery(dataElements, 1, false);
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, 1, false);
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, 1, false);
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+ // Run tests without using the embedded selector
+ SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+ BaseTests.testDNSHostnameQuery(dataElements, 1, false);
+ BaseTests.testSRCIPQuery(dataElements, 2);
+ BaseTests.testDNSIPQuery(dataElements, 3);
+ BaseTests.testDNSNXDOMAINQuery(dataElementsRcode3, 4);
+
+ // Run using a false positive
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, 1, true);
+
+ // Reset the stoplist file property
+ SystemConfiguration.setProperty("pir.stopListFile", stopListFileProp);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/wideskies/standalone/StandaloneTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/wideskies/standalone/StandaloneTest.java b/src/test/java/org/apache/pirk/wideskies/standalone/StandaloneTest.java
deleted file mode 100644
index 06c2bff..0000000
--- a/src/test/java/org/apache/pirk/wideskies/standalone/StandaloneTest.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.wideskies.standalone;
-
-import java.util.ArrayList;
-
-import org.apache.pirk.schema.data.DataSchemaRegistry;
-import org.apache.pirk.schema.query.QuerySchemaRegistry;
-import org.apache.pirk.schema.query.filter.StopListFilter;
-import org.apache.pirk.test.utils.BaseTests;
-import org.apache.pirk.test.utils.Inputs;
-import org.apache.pirk.utils.SystemConfiguration;
-import org.json.simple.JSONObject;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Functional test suite for stand alone testing - non Spark applications
- * <p>
- * Tests low side module and basic encryption, decryption mechanisms
- * <p>
- * Using a fixed 8-bit data partition size (consistent with the currently codebase)
- * <p>
- * Runs with useExpLookupTable = false as generating the lookup table takes too long for normal in-memory builds
- *
- */
-public class StandaloneTest
-{
- private static final Logger logger = LoggerFactory.getLogger(StandaloneTest.class);
-
- private static final String STOPLIST_FILE = "testStopListFile";
-
- private static String stopListFileProp = null;
-
- @BeforeClass
- public static void setup() throws Exception
- {
- // Reset the schema properties and registries
- DataSchemaRegistry.clearRegistry();
- QuerySchemaRegistry.clearRegistry();
- SystemConfiguration.setProperty("data.schemas", "none");
- SystemConfiguration.setProperty("query.schemas", "none");
-
- // Create the stoplist file
- stopListFileProp = SystemConfiguration.getProperty("pir.stopListFile");
- SystemConfiguration.setProperty("pir.stopListFile", STOPLIST_FILE);
- String newSLFile = Inputs.createPIRStopList(null, false);
- SystemConfiguration.setProperty("pir.stopListFile", newSLFile);
- logger.info("stopListFileProp = " + stopListFileProp + " new prop = " + SystemConfiguration.getProperty("pir.stopListFile"));
-
- // Create data and query schemas
- Inputs.createSchemaFiles(StopListFilter.class.getName());
- }
-
- @AfterClass
- public static void teardown()
- {
- // Reset the schema properties and registries
- DataSchemaRegistry.clearRegistry();
- QuerySchemaRegistry.clearRegistry();
- SystemConfiguration.setProperty("data.schemas", "none");
- SystemConfiguration.setProperty("query.schemas", "none");
- }
-
- @Test
- public void runTests() throws Exception
- {
- ArrayList<JSONObject> dataElements = Inputs.createJSONDataElements();
- ArrayList<JSONObject> dataElementsRcode3 = Inputs.getRcode3JSONDataElements();
-
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
-
- // Run tests and use the embedded selector
- SystemConfiguration.setProperty("pirTest.embedSelector", "true");
- BaseTests.testDNSHostnameQuery(dataElements, 1, false);
- BaseTests.testSRCIPQuery(dataElements, 2);
- BaseTests.testDNSIPQuery(dataElements, 3); // numThreads % num elements to encrypt != 0
- BaseTests.testDNSNXDOMAINQuery(dataElementsRcode3, 4); // numThreads % num elements to encrypt = 0
-
- // Test embedded QuerySchema
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
- BaseTests.testDNSHostnameQuery(dataElements, 1, false);
-
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
- BaseTests.testDNSHostnameQuery(dataElements, 1, false);
-
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
- BaseTests.testDNSHostnameQuery(dataElements, 1, false);
- SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
-
- // Run tests without using the embedded selector
- SystemConfiguration.setProperty("pirTest.embedSelector", "false");
- BaseTests.testDNSHostnameQuery(dataElements, 1, false);
- BaseTests.testSRCIPQuery(dataElements, 2);
- BaseTests.testDNSIPQuery(dataElements, 3);
- BaseTests.testDNSNXDOMAINQuery(dataElementsRcode3, 4);
-
- // Run using a false positive
- SystemConfiguration.setProperty("pirTest.embedSelector", "true");
- BaseTests.testDNSHostnameQuery(dataElements, 1, true);
-
- // Reset the stoplist file property
- SystemConfiguration.setProperty("pir.stopListFile", stopListFileProp);
- }
-}
[2/6] incubator-pirk git commit: Move all tests under src/test
directory, this closes apache/incubator-pirk#66
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/schema/query/LoadQuerySchemaTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/schema/query/LoadQuerySchemaTest.java b/src/test/java/org/apache/pirk/test/schema/query/LoadQuerySchemaTest.java
new file mode 100644
index 0000000..ee683b2
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/schema/query/LoadQuerySchemaTest.java
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.schema.query;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.apache.pirk.schema.data.DataSchemaLoader;
+import org.apache.pirk.schema.data.partitioner.IPDataPartitioner;
+import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner;
+import org.apache.pirk.schema.query.QuerySchema;
+import org.apache.pirk.schema.query.QuerySchemaLoader;
+import org.apache.pirk.schema.query.QuerySchemaRegistry;
+import org.apache.pirk.schema.query.filter.StopListFilter;
+import org.apache.pirk.test.utils.Inputs;
+import org.apache.pirk.test.utils.TestUtils;
+import org.apache.pirk.utils.PIRException;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+/**
+ * Test suite for LoadQuerySchema and QuerySchema
+ */
+public class LoadQuerySchemaTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(LoadQuerySchemaTest.class);
+
+ private String querySchemaFile = "querySchemaFile";
+ private String dataSchemaName = "fakeDataSchema";
+ private String querySchemaName = "fakeQuerySchema";
+
+ private String element1 = "elementName1";
+ private String element2 = "elementName2";
+ private String element3 = "elementName3";
+ private String element4 = "elementName4";
+
+ private List<String> queryElements = Arrays.asList(element1, element2, element3);
+ private List<String> filterElements = Collections.singletonList(element2);
+
+ @Test
+ public void testGeneralSchemaLoad() throws Exception
+ {
+ logger.info("Starting testGeneralSchemaLoad: ");
+
+ // Pull off the properties and reset upon completion
+ String dataSchemasProp = SystemConfiguration.getProperty("data.schemas", "none");
+ String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
+ String stopListFileProp = SystemConfiguration.getProperty("pir.stopListFile");
+
+ // Create the stoplist file
+ createStopListFile();
+
+ // Create the data schema used and force it to load
+ try
+ {
+ createDataSchema("dataSchemaFile");
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ DataSchemaLoader.initialize();
+
+ // Create the query schema used and force it to load
+ try
+ {
+ TestUtils.createQuerySchema(querySchemaFile, querySchemaName, dataSchemaName, element4, queryElements, filterElements, StopListFilter.class.getName());
+
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ QuerySchemaLoader.initialize();
+
+ // Check the entries
+ QuerySchema qSchema = QuerySchemaRegistry.get(querySchemaName);
+
+ assertEquals(querySchemaName, qSchema.getSchemaName());
+ assertEquals(dataSchemaName, qSchema.getDataSchemaName());
+ assertEquals(element4, qSchema.getSelectorName());
+
+ assertEquals(StopListFilter.class.getName(), qSchema.getFilterTypeName());
+ if (!(qSchema.getFilter() instanceof StopListFilter))
+ {
+ fail("Filter class instance must be StopListFilter");
+ }
+
+ assertEquals(3, qSchema.getElementNames().size());
+ for (String item : qSchema.getElementNames())
+ {
+ if (!(item.equals(element1) || item.equals(element2) || item.equals(element3)))
+ {
+ fail("elementNames: item = " + item + " must equal one of: " + element1 + ", " + element2 + ", or " + element3);
+ }
+ }
+ assertEquals(1, qSchema.getFilteredElementNames().size());
+ for (String item : qSchema.getFilteredElementNames())
+ {
+ if (!item.equals(element2))
+ {
+ fail("filterElementNames: item = " + item + " must equal " + element2);
+ }
+ }
+
+ // one string, array IPs, array integers
+ int stringSize = Integer.parseInt(SystemConfiguration.getProperty("pir.stringBits"));
+ int arrayMult = Integer.parseInt(SystemConfiguration.getProperty("pir.numReturnArrayElements"));
+ int dataElementSize = stringSize + 32 * arrayMult + 32 * arrayMult;
+ assertEquals(dataElementSize, qSchema.getDataElementSize());
+
+ // Reset original query and data schema properties
+ SystemConfiguration.setProperty("data.schemas", dataSchemasProp);
+ SystemConfiguration.setProperty("query.schemas", querySchemasProp);
+ SystemConfiguration.setProperty("pir.stopListFile", stopListFileProp);
+
+ // Force the query and data schemas to load their original values
+ if (!dataSchemasProp.equals("none"))
+ {
+ DataSchemaLoader.initialize();
+ }
+
+ if (!querySchemasProp.equals("none"))
+ {
+ QuerySchemaLoader.initialize();
+ }
+
+ logger.info("Finished testGeneralSchemaLoad: ");
+ }
+
+ @Test
+ public void testUnknownFilterClass() throws Exception
+ {
+ // Pull off the properties and reset upon completion
+ String dataSchemasProp = SystemConfiguration.getProperty("data.schemas", "none");
+ String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
+
+ // Create the data schema used and force it to load
+ try
+ {
+ createDataSchema("dataSchemaFile");
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ DataSchemaLoader.initialize();
+
+ // Create the query schema used and force it to load
+ try
+ {
+ TestUtils.createQuerySchema(querySchemaFile, querySchemaName, dataSchemaName, "nonExistentElement", queryElements, filterElements, "bogusFilterClass");
+
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ try
+ {
+ QuerySchemaLoader.initialize();
+ fail("QuerySchemaLoader did not throw exception for bogus filter class");
+ } catch (Exception ignore)
+ {}
+
+ // Reset original query and data schema properties
+ SystemConfiguration.setProperty("data.schemas", dataSchemasProp);
+ SystemConfiguration.setProperty("query.schemas", querySchemasProp);
+
+ // Force the query and data schemas to load their original values
+ if (!dataSchemasProp.equals("none"))
+ {
+ DataSchemaLoader.initialize();
+ }
+
+ if (!querySchemasProp.equals("none"))
+ {
+ QuerySchemaLoader.initialize();
+ }
+
+ logger.info("Finished testFunkyFilterScenarios");
+ }
+
+ @Test
+ public void testDataSchemaDoesNotExist() throws Exception
+ {
+ logger.info("Starting testDataSchemaDoesNotExist: ");
+
+ // Pull off the properties and reset upon completion
+ String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
+
+ // Create the query schema used and force it to load
+ try
+ {
+ TestUtils.createQuerySchema(querySchemaFile, querySchemaName, dataSchemaName, element4, queryElements, filterElements, null);
+
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ try
+ {
+ QuerySchemaLoader.initialize();
+ fail("QuerySchemaLoader did not throw exception for non-existent DataSchema");
+ } catch (Exception ignore)
+ {}
+
+ // Reset original query properties and force to load
+ SystemConfiguration.setProperty("query.schemas", querySchemasProp);
+ if (!querySchemasProp.equals("none"))
+ {
+ QuerySchemaLoader.initialize();
+ }
+
+ logger.info("Finished testDataSchemaDoesNotExist ");
+ }
+
+ @Test
+ public void testSelectorDoesNotExistInDataSchema() throws Exception
+ {
+ logger.info("Starting testSelectorDoesNotExistInDataSchema: ");
+
+ // Pull off the properties and reset upon completion
+ String dataSchemasProp = SystemConfiguration.getProperty("data.schemas", "none");
+ String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
+
+ // Create the data schema used and force it to load
+ try
+ {
+ createDataSchema("dataSchemaFile");
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ DataSchemaLoader.initialize();
+
+ // Create the query schema used and force it to load
+ try
+ {
+ TestUtils.createQuerySchema(querySchemaFile, querySchemaName, dataSchemaName, "nonExistentElement", queryElements, filterElements,
+ StopListFilter.class.getName());
+
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ try
+ {
+ QuerySchemaLoader.initialize();
+ fail("QuerySchemaLoader did not throw exception for non-existent selectorName");
+ } catch (Exception ignore)
+ {}
+
+ // Reset original query and data schema properties
+ SystemConfiguration.setProperty("data.schemas", dataSchemasProp);
+ SystemConfiguration.setProperty("query.schemas", querySchemasProp);
+
+ // Force the query and data schemas to load their original values
+ if (!dataSchemasProp.equals("none"))
+ {
+ DataSchemaLoader.initialize();
+ }
+
+ if (!querySchemasProp.equals("none"))
+ {
+ QuerySchemaLoader.initialize();
+ }
+
+ logger.info("Finished testSelectorDoesNotExistInDataSchema ");
+ }
+
+ // Create the stoplist file and alter the properties accordingly
+ private void createStopListFile() throws IOException, PIRException
+ {
+ SystemConfiguration.setProperty("pir.stopListFile", "testStopListFile");
+ String newSLFile = Inputs.createPIRStopList(null, false);
+ SystemConfiguration.setProperty("pir.stopListFile", newSLFile);
+ }
+
+ // Create the test data schema file
+ private void createDataSchema(String schemaFile) throws IOException
+ {
+ // Create a temporary file for the test schema, set in the properties
+ File file = File.createTempFile(schemaFile, ".xml");
+ file.deleteOnExit();
+ logger.info("file = " + file.toString());
+ SystemConfiguration.setProperty("data.schemas", file.toString());
+
+ // Write to the file
+ try
+ {
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+ Document doc = dBuilder.newDocument();
+
+ // root element
+ Element rootElement = doc.createElement("schema");
+ doc.appendChild(rootElement);
+
+ // Add the schemaName
+ Element schemaNameElement = doc.createElement("schemaName");
+ schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
+ rootElement.appendChild(schemaNameElement);
+
+ // Add the elements
+ // element1 -- single String
+ TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.STRING, "false", PrimitiveTypePartitioner.class.getName());
+
+ // element2 - -- array of Integers
+ TestUtils.addElement(doc, rootElement, element2, PrimitiveTypePartitioner.INT, "true", PrimitiveTypePartitioner.class.getName());
+
+ // element3 -- array of IP addresses
+ TestUtils.addElement(doc, rootElement, element3, PrimitiveTypePartitioner.STRING, "true", IPDataPartitioner.class.getName());
+
+ // element4 -- single byte type
+ TestUtils.addElement(doc, rootElement, element4, PrimitiveTypePartitioner.BYTE, "false", PrimitiveTypePartitioner.class.getName());
+
+ // Write to a xml file
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ Transformer transformer = transformerFactory.newTransformer();
+ DOMSource source = new DOMSource(doc);
+ StreamResult result = new StreamResult(file);
+ transformer.transform(source, result);
+
+ // Output for testing
+ StreamResult consoleResult = new StreamResult(System.out);
+ transformer.transform(source, consoleResult);
+ System.out.println();
+
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/serialization/SerializationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/serialization/SerializationTest.java b/src/test/java/org/apache/pirk/test/serialization/SerializationTest.java
new file mode 100644
index 0000000..57fe559
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/serialization/SerializationTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.serialization;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.Random;
+
+import org.apache.pirk.serialization.JavaSerializer;
+import org.apache.pirk.serialization.JsonSerializer;
+import org.apache.pirk.serialization.Storable;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class SerializationTest
+{
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ private static JsonSerializer jsonSerializer;
+ private static JavaSerializer javaSerializer;
+
+ @BeforeClass
+ public static void setUp() throws Exception
+ {
+ jsonSerializer = new JsonSerializer();
+ javaSerializer = new JavaSerializer();
+ }
+
+ @Test
+ public void testJsonSerDe() throws Exception
+ {
+ File tempFile = folder.newFile("test-json-serialize");
+ FileOutputStream fos = new FileOutputStream(tempFile);
+ DummyRecord dummyRecord = new DummyRecord();
+
+ jsonSerializer.write(fos, dummyRecord);
+
+ FileInputStream fis = new FileInputStream(tempFile);
+ Object deserializedDummyObject = jsonSerializer.read(fis, DummyRecord.class);
+ Assert.assertEquals(dummyRecord, deserializedDummyObject);
+ }
+
+ @Test
+ public void testJavaSerDe() throws Exception
+ {
+ File tempFile = folder.newFile("test-java-serialize");
+ FileOutputStream fos = new FileOutputStream(tempFile);
+ DummyRecord dummyRecord = new DummyRecord();
+
+ javaSerializer.write(fos, new DummyRecord());
+
+ FileInputStream fis = new FileInputStream(tempFile);
+ Object deserializedDummyObject = javaSerializer.read(fis, DummyRecord.class);
+ Assert.assertTrue(deserializedDummyObject.equals(dummyRecord));
+ }
+
+ private static class DummyRecord implements Serializable, Storable
+ {
+ private int id;
+ private String message;
+ private long seed = 100L;
+
+ DummyRecord()
+ {
+ this.id = (new Random(seed)).nextInt(5);
+ this.message = "The next message id is " + id;
+ }
+
+ public int getId()
+ {
+ return id;
+ }
+
+ public void setId(int id)
+ {
+ this.id = id;
+ }
+
+ public String getMessage()
+ {
+ return message;
+ }
+
+ public void setMessage(String message)
+ {
+ this.message = message;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "DummyRecord{" + "id=" + id + ", message='" + message + '\'' + '}';
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ DummyRecord that = (DummyRecord) o;
+ return id == that.id && Objects.equals(message, that.message);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(id, message);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/utils/BaseTests.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/utils/BaseTests.java b/src/test/java/org/apache/pirk/test/utils/BaseTests.java
new file mode 100644
index 0000000..a55ed4d
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/utils/BaseTests.java
@@ -0,0 +1,643 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.utils;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.pirk.query.wideskies.QueryUtils;
+import org.apache.pirk.schema.query.QuerySchema;
+import org.apache.pirk.schema.query.QuerySchemaRegistry;
+import org.apache.pirk.schema.response.QueryResponseJSON;
+import org.apache.pirk.test.distributed.testsuite.DistTestSuite;
+import org.apache.pirk.utils.StringUtils;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Class to hold the base functional distributed tests
+ */
+public class BaseTests
+{
+ private static final Logger logger = LoggerFactory.getLogger(BaseTests.class);
+
+ public static final UUID queryIdentifier = UUID.randomUUID();
+ public static final int dataPartitionBitSize = 8;
+
+ // Selectors for domain and IP queries, queryIdentifier is the first entry for file generation
+ private static ArrayList<String> selectorsDomain = new ArrayList<>(Arrays.asList("s.t.u.net", "d.e.com", "r.r.r.r", "a.b.c.com", "something.else", "x.y.net"));
+ private static ArrayList<String> selectorsIP = new ArrayList<>(Arrays.asList("55.55.55.55", "5.6.7.8", "10.20.30.40", "13.14.15.16", "21.22.23.24"));
+
+ // Encryption variables -- Paillier mechanisms are tested in the Paillier test code, so these are fixed...
+ public static final int hashBitSize = 12;
+ public static final String hashKey = "someKey";
+ public static final int paillierBitSize = 384;
+ public static final int certainty = 128;
+
+ public static void testDNSHostnameQuery(ArrayList<JSONObject> dataElements, int numThreads, boolean testFalsePositive) throws Exception
+ {
+ testDNSHostnameQuery(dataElements, null, false, false, numThreads, testFalsePositive);
+ }
+
+ public static void testDNSHostnameQuery(List<JSONObject> dataElements, FileSystem fs, boolean isSpark, boolean isDistributed, int numThreads)
+ throws Exception
+ {
+ testDNSHostnameQuery(dataElements, fs, isSpark, isDistributed, numThreads, false);
+ }
+
+ // Query for the watched hostname occurred; ; watched value type: hostname (String)
+ public static void testDNSHostnameQuery(List<JSONObject> dataElements, FileSystem fs, boolean isSpark, boolean isDistributed, int numThreads,
+ boolean testFalsePositive) throws Exception
+ {
+ logger.info("Running testDNSHostnameQuery(): ");
+
+ QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_HOSTNAME_QUERY);
+
+ int numExpectedResults = 6;
+ List<QueryResponseJSON> results;
+ if (isDistributed)
+ {
+ results = DistTestSuite.performQuery(Inputs.DNS_HOSTNAME_QUERY, selectorsDomain, fs, isSpark, numThreads);
+ }
+ else
+ {
+ results = StandaloneQuery.performStandaloneQuery(dataElements, Inputs.DNS_HOSTNAME_QUERY, selectorsDomain, numThreads, testFalsePositive);
+ if (!testFalsePositive)
+ {
+ numExpectedResults = 7; // all 7 for non distributed case; if testFalsePositive==true, then 6
+ }
+ }
+ logger.info("results:");
+ printResultList(results);
+
+ if (isDistributed && SystemConfiguration.isSetTrue("pir.limitHitsPerSelector"))
+ {
+ // 3 elements returned - one for each qname -- a.b.c.com, d.e.com, something.else
+ if (results.size() != 3)
+ {
+ fail("results.size() = " + results.size() + " -- must equal 3");
+ }
+
+ // Check that each qname appears once in the result set
+ HashSet<String> correctQnames = new HashSet<>();
+ correctQnames.add("a.b.c.com");
+ correctQnames.add("d.e.com");
+ correctQnames.add("something.else");
+
+ HashSet<String> resultQnames = new HashSet<>();
+ for (QueryResponseJSON qrJSON : results)
+ {
+ resultQnames.add((String) qrJSON.getValue(Inputs.QNAME));
+ }
+
+ if (correctQnames.size() != resultQnames.size())
+ {
+ fail("correctQnames.size() = " + correctQnames.size() + " != resultQnames.size() " + resultQnames.size());
+ }
+
+ for (String resultQname : resultQnames)
+ {
+ if (!correctQnames.contains(resultQname))
+ {
+ fail("correctQnames does not contain resultQname = " + resultQname);
+ }
+ }
+ }
+ else
+ {
+ if (results.size() != numExpectedResults)
+ {
+ fail("results.size() = " + results.size() + " -- must equal " + numExpectedResults);
+ }
+
+ // Number of original elements at the end of the list that we do not need to consider for hits
+ int removeTailElements = 2; // the last two data elements should not hit
+ if (testFalsePositive)
+ {
+ removeTailElements = 3;
+ }
+
+ ArrayList<QueryResponseJSON> correctResults = new ArrayList<>();
+ int i = 0;
+ while (i < (dataElements.size() - removeTailElements))
+ {
+ JSONObject dataMap = dataElements.get(i);
+
+ boolean addElement = true;
+ if (isDistributed && dataMap.get(Inputs.RCODE).toString().equals("3"))
+ {
+ addElement = false;
+ }
+ if (addElement)
+ {
+ QueryResponseJSON wlJSON = new QueryResponseJSON();
+ wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier.toString());
+ wlJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_HOSTNAME_QUERY);
+ wlJSON.setMapping(Inputs.DATE, dataMap.get(Inputs.DATE));
+ wlJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP));
+ wlJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
+ wlJSON.setMapping(Inputs.QNAME, dataMap.get(Inputs.QNAME)); // this gets re-embedded as the original selector after decryption
+ wlJSON.setMapping(Inputs.QTYPE, parseShortArray(dataMap, Inputs.QTYPE));
+ wlJSON.setMapping(Inputs.RCODE, dataMap.get(Inputs.RCODE));
+ wlJSON.setMapping(Inputs.IPS, parseArray(dataMap, Inputs.IPS, true));
+ wlJSON.setMapping(QueryResponseJSON.SELECTOR, QueryUtils.getSelectorByQueryTypeJSON(qSchema, dataMap));
+ correctResults.add(wlJSON);
+ }
+ ++i;
+ }
+ logger.info("correctResults: ");
+ printResultList(correctResults);
+
+ if (results.size() != correctResults.size())
+ {
+ logger.info("correctResults:");
+ printResultList(correctResults);
+ fail("results.size() = " + results.size() + " != correctResults.size() = " + correctResults.size());
+ }
+ for (QueryResponseJSON result : results)
+ {
+ if (!compareResultArray(correctResults, result))
+ {
+ fail("correctResults does not contain result = " + result.toString());
+ }
+ }
+ }
+ logger.info("Completed testDNSHostnameQuery(): ");
+ }
+
+ public static void testDNSIPQuery(ArrayList<JSONObject> dataElements, int numThreads) throws Exception
+ {
+ testDNSIPQuery(dataElements, null, false, false, numThreads);
+ }
+
+ // The watched IP address was detected in the response to a query; watched value type: IP address (String)
+ public static void testDNSIPQuery(List<JSONObject> dataElements, FileSystem fs, boolean isSpark, boolean isDistributed, int numThreads) throws Exception
+ {
+ logger.info("Running testDNSIPQuery(): ");
+
+ QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_IP_QUERY);
+ List<QueryResponseJSON> results;
+
+ if (isDistributed)
+ {
+ results = DistTestSuite.performQuery(Inputs.DNS_IP_QUERY, selectorsIP, fs, isSpark, numThreads);
+
+ if (results.size() != 5)
+ {
+ fail("results.size() = " + results.size() + " -- must equal 5");
+ }
+ }
+ else
+ {
+ results = StandaloneQuery.performStandaloneQuery(dataElements, Inputs.DNS_IP_QUERY, selectorsIP, numThreads, false);
+
+ if (results.size() != 6)
+ {
+ fail("results.size() = " + results.size() + " -- must equal 6");
+ }
+ }
+ printResultList(results);
+
+ ArrayList<QueryResponseJSON> correctResults = new ArrayList<>();
+ int i = 0;
+ while (i < (dataElements.size() - 3)) // last three data elements not hit - one on stoplist, two don't match selectors
+ {
+ JSONObject dataMap = dataElements.get(i);
+
+ boolean addElement = true;
+ if (isDistributed && dataMap.get(Inputs.RCODE).toString().equals("3"))
+ {
+ addElement = false;
+ }
+ if (addElement)
+ {
+ QueryResponseJSON wlJSON = new QueryResponseJSON();
+ wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier);
+ wlJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_IP_QUERY);
+ wlJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP));
+ wlJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
+ wlJSON.setMapping(Inputs.IPS, parseArray(dataMap, Inputs.IPS, true));
+ wlJSON.setMapping(QueryResponseJSON.SELECTOR, QueryUtils.getSelectorByQueryTypeJSON(qSchema, dataMap));
+ correctResults.add(wlJSON);
+ }
+ ++i;
+ }
+ if (results.size() != correctResults.size())
+ {
+ logger.info("correctResults:");
+ printResultList(correctResults);
+ fail("results.size() = " + results.size() + " != correctResults.size() = " + correctResults.size());
+ }
+ for (QueryResponseJSON result : results)
+ {
+ if (!compareResultArray(correctResults, result))
+ {
+ fail("correctResults does not contain result = " + result.toString());
+ }
+ }
+ logger.info("Completed testDNSIPQuery(): ");
+ }
+
+ public static void testDNSNXDOMAINQuery(ArrayList<JSONObject> dataElements, int numThreads) throws Exception
+ {
+ testDNSNXDOMAINQuery(dataElements, null, false, false, numThreads);
+ }
+
+ // A query that returned an nxdomain response was made for the watched hostname; watched value type: hostname (String)
+ public static void testDNSNXDOMAINQuery(List<JSONObject> dataElements, FileSystem fs, boolean isSpark, boolean isDistributed, int numThreads)
+ throws Exception
+ {
+ logger.info("Running testDNSNXDOMAINQuery(): ");
+
+ QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_NXDOMAIN_QUERY);
+ List<QueryResponseJSON> results;
+
+ if (isDistributed)
+ {
+ results = DistTestSuite.performQuery(Inputs.DNS_NXDOMAIN_QUERY, selectorsDomain, fs, isSpark, numThreads);
+ }
+ else
+ {
+ results = StandaloneQuery.performStandaloneQuery(dataElements, Inputs.DNS_NXDOMAIN_QUERY, selectorsDomain, numThreads, false);
+ }
+ printResultList(results);
+
+ if (results.size() != 1)
+ {
+ fail("results.size() = " + results.size() + " -- must equal 1");
+ }
+
+ ArrayList<QueryResponseJSON> correctResults = new ArrayList<>();
+ int i = 0;
+ while (i < dataElements.size())
+ {
+ JSONObject dataMap = dataElements.get(i);
+
+ if (dataMap.get(Inputs.RCODE).toString().equals("3"))
+ {
+ QueryResponseJSON wlJSON = new QueryResponseJSON();
+ wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier);
+ wlJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_NXDOMAIN_QUERY);
+ wlJSON.setMapping(Inputs.QNAME, dataMap.get(Inputs.QNAME)); // this gets re-embedded as the original selector after decryption
+ wlJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
+ wlJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP));
+ wlJSON.setMapping(QueryResponseJSON.SELECTOR, QueryUtils.getSelectorByQueryTypeJSON(qSchema, dataMap));
+ correctResults.add(wlJSON);
+ }
+ ++i;
+ }
+ if (results.size() != correctResults.size())
+ {
+ logger.info("correctResults:");
+ printResultList(correctResults);
+ fail("results.size() = " + results.size() + " != correctResults.size() = " + correctResults.size());
+ }
+ for (QueryResponseJSON result : results)
+ {
+ if (!compareResultArray(correctResults, result))
+ {
+ fail("correctResults does not contain result = " + result.toString());
+ }
+ }
+ logger.info("Completed testDNSNXDOMAINQuery(): ");
+ }
+
+ public static void testSRCIPQuery(ArrayList<JSONObject> dataElements, int numThreads) throws Exception
+ {
+ testSRCIPQuery(dataElements, null, false, false, numThreads);
+ }
+
+ // Query for responses from watched srcIPs
+ public static void testSRCIPQuery(List<JSONObject> dataElements, FileSystem fs, boolean isSpark, boolean isDistributed, int numThreads) throws Exception
+ {
+ logger.info("Running testSRCIPQuery(): ");
+
+ QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_SRCIP_QUERY);
+ List<QueryResponseJSON> results;
+
+ int removeTailElements = 0;
+ int numExpectedResults = 1;
+ if (isDistributed)
+ {
+ results = DistTestSuite.performQuery(Inputs.DNS_SRCIP_QUERY, selectorsIP, fs, isSpark, numThreads);
+ removeTailElements = 2; // The last two elements are on the distributed stoplist
+ }
+ else
+ {
+ numExpectedResults = 3;
+ results = StandaloneQuery.performStandaloneQuery(dataElements, Inputs.DNS_SRCIP_QUERY, selectorsIP, numThreads, false);
+ }
+ printResultList(results);
+
+ if (results.size() != numExpectedResults)
+ {
+ fail("results.size() = " + results.size() + " -- must equal " + numExpectedResults);
+ }
+
+ ArrayList<QueryResponseJSON> correctResults = new ArrayList<>();
+ int i = 0;
+ while (i < (dataElements.size() - removeTailElements))
+ {
+ JSONObject dataMap = dataElements.get(i);
+
+ boolean addElement = false;
+ if (dataMap.get(Inputs.SRCIP).toString().equals("55.55.55.55") || dataMap.get(Inputs.SRCIP).toString().equals("5.6.7.8"))
+ {
+ addElement = true;
+ }
+ if (addElement)
+ {
+ // Form the correct result QueryResponseJSON object
+ QueryResponseJSON qrJSON = new QueryResponseJSON();
+ qrJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier);
+ qrJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_SRCIP_QUERY);
+ qrJSON.setMapping(Inputs.QNAME, parseString(dataMap, Inputs.QNAME));
+ qrJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
+ qrJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP));
+ qrJSON.setMapping(Inputs.IPS, parseArray(dataMap, Inputs.IPS, true));
+ qrJSON.setMapping(QueryResponseJSON.SELECTOR, QueryUtils.getSelectorByQueryTypeJSON(qSchema, dataMap));
+ correctResults.add(qrJSON);
+ }
+ ++i;
+ }
+ logger.info("correctResults:");
+ printResultList(correctResults);
+
+ if (results.size() != correctResults.size())
+ {
+ logger.info("correctResults:");
+ printResultList(correctResults);
+ fail("results.size() = " + results.size() + " != correctResults.size() = " + correctResults.size());
+ }
+ for (QueryResponseJSON result : results)
+ {
+ if (!compareResultArray(correctResults, result))
+ {
+ fail("correctResults does not contain result = " + result.toString());
+ }
+ }
+ logger.info("Completed testSRCIPQuery(): ");
+ }
+
+ // Query for responses from watched srcIPs
+ public static void testSRCIPQueryNoFilter(List<JSONObject> dataElements, FileSystem fs, boolean isSpark, boolean isDistributed, int numThreads)
+ throws Exception
+ {
+ logger.info("Running testSRCIPQueryNoFilter(): ");
+
+ QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_SRCIP_QUERY_NO_FILTER);
+ List<QueryResponseJSON> results;
+
+ int numExpectedResults = 3;
+ if (isDistributed)
+ {
+ results = DistTestSuite.performQuery(Inputs.DNS_SRCIP_QUERY_NO_FILTER, selectorsIP, fs, isSpark, numThreads);
+ }
+ else
+ {
+ results = StandaloneQuery.performStandaloneQuery(dataElements, Inputs.DNS_SRCIP_QUERY_NO_FILTER, selectorsIP, numThreads, false);
+ }
+ printResultList(results);
+
+ if (results.size() != numExpectedResults)
+ {
+ fail("results.size() = " + results.size() + " -- must equal " + numExpectedResults);
+ }
+
+ ArrayList<QueryResponseJSON> correctResults = new ArrayList<>();
+ int i = 0;
+ while (i < dataElements.size())
+ {
+ JSONObject dataMap = dataElements.get(i);
+
+ boolean addElement = false;
+ if (dataMap.get(Inputs.SRCIP).toString().equals("55.55.55.55") || dataMap.get(Inputs.SRCIP).toString().equals("5.6.7.8"))
+ {
+ addElement = true;
+ }
+ if (addElement)
+ {
+ // Form the correct result QueryResponseJSON object
+ QueryResponseJSON qrJSON = new QueryResponseJSON();
+ qrJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier);
+ qrJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_SRCIP_QUERY_NO_FILTER);
+ qrJSON.setMapping(Inputs.QNAME, parseString(dataMap, Inputs.QNAME));
+ qrJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
+ qrJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP));
+ qrJSON.setMapping(Inputs.IPS, parseArray(dataMap, Inputs.IPS, true));
+ qrJSON.setMapping(QueryResponseJSON.SELECTOR, QueryUtils.getSelectorByQueryTypeJSON(qSchema, dataMap));
+ correctResults.add(qrJSON);
+ }
+ ++i;
+ }
+ logger.info("correctResults:");
+ printResultList(correctResults);
+
+ if (results.size() != correctResults.size())
+ {
+ logger.info("correctResults:");
+ printResultList(correctResults);
+ fail("results.size() = " + results.size() + " != correctResults.size() = " + correctResults.size());
+ }
+ for (QueryResponseJSON result : results)
+ {
+ if (!compareResultArray(correctResults, result))
+ {
+ fail("correctResults does not contain result = " + result.toString());
+ }
+ }
+ logger.info("Completed testSRCIPQueryNoFilter(): ");
+ }
+
+ @SuppressWarnings("unchecked")
+ // Method to convert a ArrayList<String> into the correct (padded) returned ArrayList
+ private static ArrayList<String> parseArray(JSONObject dataMap, String fieldName, boolean isIP)
+ {
+ ArrayList<String> retArray = new ArrayList<>();
+
+ ArrayList<String> values;
+ if (dataMap.get(fieldName) instanceof ArrayList)
+ {
+ values = (ArrayList<String>) dataMap.get(fieldName);
+ }
+ else
+ {
+ values = StringUtils.jsonArrayStringToArrayList((String) dataMap.get(fieldName));
+ }
+
+ int numArrayElementsToReturn = SystemConfiguration.getIntProperty("pir.numReturnArrayElements", 1);
+ for (int i = 0; i < numArrayElementsToReturn; ++i)
+ {
+ if (i < values.size())
+ {
+ retArray.add(values.get(i));
+ }
+ else if (isIP)
+ {
+ retArray.add("0.0.0.0");
+ }
+ else
+ {
+ retArray.add("0");
+ }
+ }
+
+ return retArray;
+ }
+
+ // Method to convert a ArrayList<Short> into the correct (padded) returned ArrayList
+ private static ArrayList<Short> parseShortArray(JSONObject dataMap, String fieldName)
+ {
+ ArrayList<Short> retArray = new ArrayList<>();
+
+ ArrayList<Short> values = (ArrayList<Short>) dataMap.get(fieldName);
+
+ int numArrayElementsToReturn = SystemConfiguration.getIntProperty("pir.numReturnArrayElements", 1);
+ for (int i = 0; i < numArrayElementsToReturn; ++i)
+ {
+ if (i < values.size())
+ {
+ retArray.add(values.get(i));
+ }
+ else
+ {
+ retArray.add((short) 0);
+ }
+ }
+
+ return retArray;
+ }
+
+ // Method to convert the String field value to the correct returned substring
+ private static String parseString(JSONObject dataMap, String fieldName)
+ {
+ String ret;
+
+ String element = (String) dataMap.get(fieldName);
+ int numParts = Integer.parseInt(SystemConfiguration.getProperty("pir.stringBits")) / dataPartitionBitSize;
+ int len = numParts;
+ if (element.length() < numParts)
+ {
+ len = element.length();
+ }
+ ret = new String(element.getBytes(), 0, len);
+
+ return ret;
+ }
+
+ // Method to determine whether or not the correctResults contains an object equivalent to
+ // the given result
+ private static boolean compareResultArray(ArrayList<QueryResponseJSON> correctResults, QueryResponseJSON result)
+ {
+ boolean equivalent = false;
+
+ for (QueryResponseJSON correct : correctResults)
+ {
+ equivalent = compareResults(correct, result);
+ if (equivalent)
+ {
+ break;
+ }
+ }
+
+ return equivalent;
+ }
+
+ // Method to test the equivalence of two test results
+ private static boolean compareResults(QueryResponseJSON r1, QueryResponseJSON r2)
+ {
+ boolean equivalent = true;
+
+ JSONObject jsonR1 = r1.getJSONObject();
+ JSONObject jsonR2 = r2.getJSONObject();
+
+ Set<String> r1KeySet = jsonR1.keySet();
+ Set<String> r2KeySet = jsonR2.keySet();
+ if (!r1KeySet.equals(r2KeySet))
+ {
+ equivalent = false;
+ }
+ if (equivalent)
+ {
+ for (String key : r1KeySet)
+ {
+ if (key.equals(Inputs.QTYPE) || key.equals(Inputs.IPS)) // array types
+ {
+ HashSet<String> set1 = getSetFromList(jsonR1.get(key));
+ HashSet<String> set2 = getSetFromList(jsonR2.get(key));
+
+ if (!set1.equals(set2))
+ {
+ equivalent = false;
+ }
+ }
+ else
+ {
+ if (!(jsonR1.get(key).toString()).equals(jsonR2.get(key).toString()))
+ {
+ equivalent = false;
+ }
+ }
+ }
+ }
+ return equivalent;
+ }
+
+ // Method to pull the elements of a list (either an ArrayList or JSONArray) into a HashSet
+ private static HashSet<String> getSetFromList(Object list)
+ {
+ HashSet<String> set = new HashSet<>();
+
+ if (list instanceof ArrayList)
+ {
+ for (Object obj : (ArrayList) list)
+ {
+ set.add(obj.toString());
+ }
+ }
+ else
+ // JSONArray
+ {
+ for (Object obj : (JSONArray) list)
+ {
+ set.add(obj.toString());
+ }
+ }
+
+ return set;
+ }
+
+ private static void printResultList(List<QueryResponseJSON> list)
+ {
+ for (QueryResponseJSON obj : list)
+ {
+ logger.info(obj.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/utils/Inputs.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/utils/Inputs.java b/src/test/java/org/apache/pirk/test/utils/Inputs.java
new file mode 100644
index 0000000..10c1386
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/utils/Inputs.java
@@ -0,0 +1,606 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.pirk.schema.data.DataSchemaLoader;
+import org.apache.pirk.schema.data.partitioner.IPDataPartitioner;
+import org.apache.pirk.schema.data.partitioner.ISO8601DatePartitioner;
+import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner;
+import org.apache.pirk.schema.query.QuerySchemaLoader;
+import org.apache.pirk.test.distributed.DistributedTestDriver;
+import org.apache.pirk.utils.HDFS;
+import org.apache.pirk.utils.PIRException;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+/**
+ * Input files for distributed testing
+ *
+ */
+public class Inputs
+{
+ private static final Logger logger = LoggerFactory.getLogger(Inputs.class);
+
+ // Test data schema fields
+ public static final String DATE = "date";
+ public static final String QNAME = "qname";
+ public static final String SRCIP = "src_ip";
+ public static final String DSTIP = "dest_ip";
+ public static final String QTYPE = "qtype";
+ public static final String RCODE = "rcode";
+ public static final String IPS = "ip";
+
+ // Test query types
+ public static final String DNS_HOSTNAME_QUERY = "dns-hostname-query"; // Query for the watched hostnames occurred; ; watched value type -- hostname
+ public static final String DNS_IP_QUERY = "dns-ip-query"; // The watched IP address(es) were detected in the response to a query; watched value type -- IP in
+ // IPS field (resolution IP)
+ public static final String DNS_NXDOMAIN_QUERY = "dns-nxdomain-query"; // Query for nxdomain responses that were made for watched qnames
+ public static final String DNS_SRCIP_QUERY = "dns-srcip-query"; // Query for responses from watched srcIPs
+ public static final String DNS_SRCIP_QUERY_NO_FILTER = "dns-srcip-query-no-filter"; // Query for responses from watched srcIPs, no data filter used
+
+ // Test query type files - localfs
+ public static final String DNS_HOSTNAME_QUERY_FILE = DNS_HOSTNAME_QUERY + "_file";
+ public static final String DNS_IP_QUERY_FILE = DNS_IP_QUERY + "_file";
+ public static final String DNS_NXDOMAIN_QUERY_FILE = DNS_NXDOMAIN_QUERY + "_file";
+ public static final String DNS_SRCIP_QUERY_FILE = DNS_SRCIP_QUERY + "_file";
+ public static final String DNS_SRCIP_QUERY_NO_FILTER_FILE = DNS_SRCIP_QUERY_NO_FILTER + "_file";
+
+ // Test query files hdfs
+ public static final String DNS_HOSTNAME_QUERY_FILE_HDFS = "/tmp/" + DNS_HOSTNAME_QUERY + "_file";
+ public static final String DNS_IP_QUERY_FILE_HDFS = "/tmp/" + DNS_IP_QUERY + "_file";
+ public static final String DNS_NXDOMAIN_QUERY_FILE_HDFS = "/tmp/" + DNS_NXDOMAIN_QUERY + "_file";
+ public static final String DNS_SRCIP_QUERY_FILE_HDFS = "/tmp/" + DNS_SRCIP_QUERY + "_file";
+ public static final String DNS_SRCIP_QUERY_NO_FILTER_FILE_HDFS = "/tmp/" + DNS_SRCIP_QUERY_NO_FILTER + "_file";
+
+ // Combined query file strings -- used to set properties
+ public static final String LOCALFS_QUERY_FILES = DNS_HOSTNAME_QUERY_FILE + "," + DNS_IP_QUERY_FILE + "," + DNS_NXDOMAIN_QUERY_FILE + ","
+ + DNS_SRCIP_QUERY_FILE + "," + DNS_SRCIP_QUERY_NO_FILTER_FILE;
+
+ public static final String HDFS_QUERY_FILES = DNS_HOSTNAME_QUERY_FILE_HDFS + "," + DNS_IP_QUERY_FILE_HDFS + "," + DNS_NXDOMAIN_QUERY_FILE_HDFS + ","
+ + DNS_SRCIP_QUERY_FILE_HDFS + "," + DNS_SRCIP_QUERY_NO_FILTER_FILE_HDFS;
+
+ // Test data schema files -- localFS and hdfs
+ public static final String TEST_DATA_SCHEMA_NAME = "testDataSchema";
+ public static final String DATA_SCHEMA_FILE_LOCALFS = "testDataSchemaFile";
+ public static final String DATA_SCHEMA_FILE_HDFS = "/tmp/testDataSchemaFile.xml";
+
+ /**
+ * Delete the ElasticSearch indices that was used for functional testing
+ */
+ public static void deleteESInput()
+ {
+ String esPIRIndex = SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_NODES_PROPERTY) + ":"
+ + SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_PORT_PROPERTY) + "/"
+ + SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_INDEX_PROPERTY);
+ logger.info("ES input being deleted at " + esPIRIndex);
+
+ ProcessBuilder pDeletePIR = new ProcessBuilder("curl", "-XDELETE", esPIRIndex);
+ try
+ {
+ TestUtils.executeCommand(pDeletePIR);
+ logger.info("ES input deleted!");
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Creates PIR JSON input
+ */
+ @SuppressWarnings("unchecked")
+ public static ArrayList<JSONObject> createJSONDataElements()
+ {
+ ArrayList<JSONObject> dataElementsJSON = new ArrayList<>();
+
+ JSONObject jsonObj1 = new JSONObject();
+ jsonObj1.put(DATE, "2016-02-20T23:29:05.000Z");
+ jsonObj1.put(QNAME, "a.b.c.com"); // hits on domain selector
+ jsonObj1.put(SRCIP, "55.55.55.55"); // hits on IP selector
+ jsonObj1.put(DSTIP, "1.2.3.6");
+ jsonObj1.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
+ jsonObj1.put(RCODE, 0);
+ jsonObj1.put(IPS, new ArrayList<>(Arrays.asList("10.20.30.40", "10.20.30.60")));
+
+ dataElementsJSON.add(jsonObj1);
+
+ JSONObject jsonObj2 = new JSONObject();
+ jsonObj2.put(DATE, "2016-02-20T23:29:06.000Z");
+ jsonObj2.put(QNAME, "d.e.com");
+ jsonObj2.put(SRCIP, "127.128.129.130");
+ jsonObj2.put(DSTIP, "1.2.3.4");
+ jsonObj2.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
+ jsonObj2.put(RCODE, 0);
+ jsonObj2.put(IPS, new ArrayList<>(Collections.singletonList("5.6.7.8")));
+
+ dataElementsJSON.add(jsonObj2);
+
+ JSONObject jsonObj3 = new JSONObject();
+ jsonObj3.put(DATE, "2016-02-20T23:29:07.000Z");
+ jsonObj3.put(QNAME, "d.e.com");
+ jsonObj3.put(SRCIP, "131.132.133.134");
+ jsonObj3.put(DSTIP, "9.10.11.12");
+ jsonObj3.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
+ jsonObj3.put(RCODE, 0);
+ jsonObj3.put(IPS, new ArrayList<>(Collections.singletonList("13.14.15.16")));
+
+ dataElementsJSON.add(jsonObj3);
+
+ JSONObject jsonObj4 = new JSONObject();
+ jsonObj4.put(DATE, "2016-02-20T23:29:08.000Z");
+ jsonObj4.put(QNAME, "d.e.com");
+ jsonObj4.put(SRCIP, "135.136.137.138");
+ jsonObj4.put(DSTIP, "17.18.19.20");
+ jsonObj4.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
+ jsonObj4.put(RCODE, 3);
+ jsonObj4.put(IPS, new ArrayList<>(Collections.singletonList("21.22.23.24")));
+
+ dataElementsJSON.add(jsonObj4);
+
+ JSONObject jsonObj5 = new JSONObject();
+ jsonObj5.put(DATE, "2016-02-20T23:29:09.000Z");
+ jsonObj5.put(QNAME, "d.e.com");
+ jsonObj5.put(SRCIP, "139.140.141.142");
+ jsonObj5.put(DSTIP, "25.26.27.28");
+ jsonObj5.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
+ jsonObj5.put(RCODE, 0);
+ jsonObj5.put(IPS, new ArrayList<>(Collections.singletonList("5.6.7.8")));
+
+ dataElementsJSON.add(jsonObj5);
+
+ JSONObject jsonObj6 = new JSONObject();
+ jsonObj6.put(DATE, "2016-02-20T23:29:10.000Z");
+ jsonObj6.put(QNAME, "d.e.com");
+ jsonObj6.put(SRCIP, "143.144.145.146");
+ jsonObj6.put(DSTIP, "33.34.35.36");
+ jsonObj6.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
+ jsonObj6.put(RCODE, 0);
+ jsonObj6.put(IPS, new ArrayList<>(Collections.singletonList("5.6.7.8")));
+
+ dataElementsJSON.add(jsonObj6);
+
+ JSONObject jsonObj7 = new JSONObject();
+ jsonObj7.put(DATE, "2016-02-20T23:29:11.000Z");
+ jsonObj7.put(QNAME, "something.else");
+ jsonObj7.put(SRCIP, "1.1.1.1");
+ jsonObj7.put(DSTIP, "2.2.2.2");
+ jsonObj7.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
+ jsonObj7.put(RCODE, 0);
+ jsonObj7.put(IPS, new ArrayList<>(Collections.singletonList("3.3.3.3")));
+
+ dataElementsJSON.add(jsonObj7);
+
+ // This should never be returned - doesn't hit on any domain selectors
+ // resolution ip on stoplist
+ JSONObject jsonObj8 = new JSONObject();
+ jsonObj8.put(DATE, "2016-02-20T23:29:12.000Z");
+ jsonObj8.put(QNAME, "something.else2");
+ jsonObj8.put(SRCIP, "5.6.7.8");
+ jsonObj8.put(DSTIP, "2.2.2.22");
+ jsonObj8.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
+ jsonObj8.put(RCODE, 0);
+ jsonObj8.put(IPS, new ArrayList<>(Collections.singletonList("3.3.3.132")));
+
+ dataElementsJSON.add(jsonObj8);
+
+ // This should never be returned in distributed case -- domain and resolution ip on stoplist
+ JSONObject jsonObj9 = new JSONObject();
+ jsonObj9.put(DATE, "2016-02-20T23:29:13.000Z");
+ jsonObj9.put(QNAME, "something.else.on.stoplist");
+ jsonObj9.put(SRCIP, "55.55.55.55");
+ jsonObj9.put(DSTIP, "2.2.2.232");
+ jsonObj9.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
+ jsonObj9.put(RCODE, 0);
+ jsonObj9.put(IPS, new ArrayList<>(Collections.singletonList("3.3.3.132")));
+
+ dataElementsJSON.add(jsonObj9);
+
+ return dataElementsJSON;
+ }
+
+ /**
+ * Creates an ArrayList of JSONObjects with RCODE value of 3
+ */
+ @SuppressWarnings("unchecked")
+ public static ArrayList<JSONObject> getRcode3JSONDataElements()
+ {
+ ArrayList<JSONObject> dataElementsJSON = new ArrayList<>();
+
+ JSONObject jsonObj4 = new JSONObject();
+ jsonObj4.put(DATE, "2016-02-20T23:29:08.000Z");
+ jsonObj4.put(QNAME, "d.e.com");
+ jsonObj4.put(SRCIP, "135.136.137.138");
+ jsonObj4.put(DSTIP, "17.18.19.20");
+ jsonObj4.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
+ jsonObj4.put(RCODE, 3);
+ jsonObj4.put(IPS, new ArrayList<>(Collections.singletonList("21.22.23.24")));
+
+ dataElementsJSON.add(jsonObj4);
+
+ return dataElementsJSON;
+ }
+
+ /**
+ * Creates PIR JSON input and writes to hdfs
+ */
+ public static List<JSONObject> createPIRJSONInput(FileSystem fs)
+ {
+ String inputJSONFile = SystemConfiguration.getProperty(DistributedTestDriver.JSON_PIR_INPUT_FILE_PROPERTY);
+ logger.info("PIR JSON input being created at " + inputJSONFile);
+
+ List<JSONObject> dataElementsJSON = createJSONDataElements();
+
+ HDFS.writeFile(dataElementsJSON, fs, inputJSONFile, true);
+ logger.info("PIR JSON input successfully created!");
+
+ return dataElementsJSON;
+ }
+
+ /**
+ * Creates PIR Elasticsearch input
+ */
+ public static void createPIRESInput()
+ {
+ String esTestIndex = SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_NODES_PROPERTY) + ":"
+ + SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_PORT_PROPERTY) + "/"
+ + SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_INDEX_PROPERTY);
+ String esType = SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_TYPE_PROPERTY);
+ logger.info("ES input being created at " + esTestIndex + " with type " + esType);
+
+ // Create ES Index
+ logger.info("Creating new testindex:");
+ ProcessBuilder pCreate = new ProcessBuilder("curl", "-XPUT", esTestIndex);
+ try
+ {
+ TestUtils.executeCommand(pCreate);
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ }
+
+ // Add elements
+ logger.info(" \n \n Adding elements to testindex:");
+
+ String indexTypeNum1 = esTestIndex + "/" + esType + "/1";
+ logger.info("indexTypeNum1 = " + indexTypeNum1);
+ ProcessBuilder pAdd1 = new ProcessBuilder("curl", "-XPUT", indexTypeNum1, "-d",
+ "{\"qname\":\"a.b.c.com\",\"date\":\"2016-02-20T23:29:05.000Z\",\"qtype\":[\"1\"]"
+ + ",\"rcode\":\"0\",\"src_ip\":\"55.55.55.55\",\"dest_ip\":\"1.2.3.6\"" + ",\"ip\":[\"10.20.30.40\",\"10.20.30.60\"]}");
+
+ String indexTypeNum2 = esTestIndex + "/" + esType + "/2";
+ logger.info("indexTypeNum2 = " + indexTypeNum2);
+ ProcessBuilder pAdd2 = new ProcessBuilder("curl", "-XPUT", indexTypeNum2, "-d",
+ "{\"qname\":\"d.e.com\",\"date\":\"2016-02-20T23:29:06.000Z\",\"qtype\":[\"1\"]"
+ + ",\"rcode\":\"0\",\"src_ip\":\"127.128.129.130\",\"dest_ip\":\"1.2.3.4\"" + ",\"ip\":[\"5.6.7.8\"]}");
+
+ String indexTypeNum3 = esTestIndex + "/" + esType + "/3";
+ logger.info("indexTypeNum3 = " + indexTypeNum3);
+ ProcessBuilder pAdd3 = new ProcessBuilder("curl", "-XPUT", indexTypeNum3, "-d",
+ "{\"qname\":\"d.e.com\",\"date\":\"2016-02-20T23:29:07.000Z\",\"qtype\":[\"1\"]"
+ + ",\"rcode\":\"0\",\"src_ip\":\"131.132.133.134\",\"dest_ip\":\"9.10.11.12\"" + ",\"ip\":[\"13.14.15.16\"]}");
+
+ String indexTypeNum4 = esTestIndex + "/" + esType + "/4";
+ logger.info("indexTypeNum4 = " + indexTypeNum4);
+ ProcessBuilder pAdd4 = new ProcessBuilder("curl", "-XPUT", indexTypeNum4, "-d",
+ "{\"qname\":\"d.e.com\",\"date\":\"2016-02-20T23:29:08.000Z\",\"qtype\":[\"1\"]"
+ + ",\"rcode\":\"3\",\"src_ip\":\"135.136.137.138\",\"dest_ip\":\"17.18.19.20\"" + ",\"ip\":[\"21.22.23.24\"]}");
+
+ String indexTypeNum5 = esTestIndex + "/" + esType + "/5";
+ logger.info("indexTypeNum5 = " + indexTypeNum5);
+ ProcessBuilder pAdd5 = new ProcessBuilder("curl", "-XPUT", indexTypeNum5, "-d",
+ "{\"qname\":\"d.e.com\",\"date\":\"2016-02-20T23:29:09.000Z\",\"qtype\":[\"1\"]"
+ + ",\"rcode\":\"0\",\"src_ip\":\"139.140.141.142\",\"dest_ip\":\"25.26.27.28\"" + ",\"ip\":[\"5.6.7.8\"]}");
+
+ String indexTypeNum6 = esTestIndex + "/" + esType + "/6";
+ logger.info("indexTypeNum6 = " + indexTypeNum6);
+ ProcessBuilder pAdd6 = new ProcessBuilder("curl", "-XPUT", indexTypeNum6, "-d",
+ "{\"qname\":\"d.e.com\",\"date\":\"2016-02-20T23:29:10.000Z\",\"qtype\":[\"1\"]"
+ + ",\"rcode\":\"0\",\"src_ip\":\"143.144.145.146\",\"dest_ip\":\"33.34.35.36\"" + ",\"ip\":[\"5.6.7.8\"]}");
+
+ String indexTypeNum7 = esTestIndex + "/" + esType + "/7";
+ logger.info("indexTypeNum7 = " + indexTypeNum7);
+ ProcessBuilder pAdd7 = new ProcessBuilder("curl", "-XPUT", indexTypeNum7, "-d",
+ "{\"qname\":\"something.else\",\"date\":\"2016-02-20T23:29:11.000Z\",\"qtype\":[\"1\"]"
+ + ",\"rcode\":\"0\",\"src_ip\":\"1.1.1.1\",\"dest_ip\":\"2.2.2.2\"" + ",\"ip\":[\"3.3.3.3\"]}");
+
+ // Never should be returned - doesn't hit on any selectors
+ String indexTypeNum8 = esTestIndex + "/" + esType + "/8";
+ logger.info("indexTypeNum8 = " + indexTypeNum8);
+ ProcessBuilder pAdd8 = new ProcessBuilder("curl", "-XPUT", indexTypeNum8, "-d",
+ "{\"qname\":\"something.else2\",\"date\":\"2016-02-20T23:29:12.000Z\",\"qtype\":[\"1\"]"
+ + ",\"rcode\":\"0\",\"src_ip\":\"1.1.1.12\",\"dest_ip\":\"2.2.2.22\"" + ",\"ip\":[\"3.3.3.32\"]}");
+
+ // This should never be returned -- domain on stoplist
+ String indexTypeNum9 = esTestIndex + "/" + esType + "/9";
+ logger.info("indexTypeNum9 = " + indexTypeNum9);
+ ProcessBuilder pAdd9 = new ProcessBuilder("curl", "-XPUT", indexTypeNum9, "-d",
+ "{\"qname\":\"something.else.on.stoplist\",\"date\":\"2016-02-20T23:29:13.000Z\",\"qtype\":[\"1\"]"
+ + ",\"rcode\":\"0\",\"src_ip\":\"55.55.55.55\",\"dest_ip\":\"2.2.2.232\"" + ",\"ip\":[\"3.3.3.132\"]}");
+
+ try
+ {
+ TestUtils.executeCommand(pAdd1);
+ TestUtils.executeCommand(pAdd2);
+ TestUtils.executeCommand(pAdd3);
+ TestUtils.executeCommand(pAdd4);
+ TestUtils.executeCommand(pAdd5);
+ TestUtils.executeCommand(pAdd6);
+ TestUtils.executeCommand(pAdd7);
+ TestUtils.executeCommand(pAdd8);
+ TestUtils.executeCommand(pAdd9);
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ }
+
+ // Retrieve and print all of the elements
+ for (int i = 1; i < 7; ++i)
+ {
+ logger.info("Retrieving element number = " + i + " from " + esTestIndex);
+ String elementGet = esTestIndex + "/" + esType + "/" + i;
+ logger.info("elementGet = " + elementGet);
+ ProcessBuilder pGet = new ProcessBuilder("curl", "-XGET", elementGet);
+ try
+ {
+ TestUtils.executeCommand(pGet);
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ /**
+ * Creates PIR stoplist file
+ */
+ public static String createPIRStopList(FileSystem fs, boolean hdfs) throws IOException, PIRException
+ {
+ logger.info("PIR stopList file being created");
+
+ List<String> elements = Arrays.asList("something.else.on.stoplist", "3.3.3.132");
+
+ if (hdfs)
+ {
+ String pirStopListFile = SystemConfiguration.getProperty(DistributedTestDriver.PIR_STOPLIST_FILE);
+ if (pirStopListFile == null)
+ {
+ throw new PIRException("HDFS stop list file configuration name is required.");
+ }
+ HDFS.writeFile(elements, fs, pirStopListFile, true);
+ logger.info("pirStopListFile file successfully created on hdfs!");
+ }
+
+ String prefix = SystemConfiguration.getProperty("pir.stopListFile");
+ if (prefix == null)
+ {
+ throw new PIRException("Local stop list file configuration name is required.");
+ }
+ return TestUtils.writeToTmpFile(elements, prefix, null);
+ }
+
+ /**
+ * Create and load the data and query schema files used for testing
+ */
+ public static void createSchemaFiles(String filter) throws Exception
+ {
+ createSchemaFiles(null, false, filter);
+ }
+
+ /**
+ * Create and load the data and query schema files used for testing
+ * <p>
+ * Writes both local and hdfs schema files if hdfs=true -- only updates the corresponding properties for the local files
+ */
+ public static void createSchemaFiles(FileSystem fs, boolean hdfs, String filter) throws Exception
+ {
+ // Create and load the data schema
+ if (!hdfs)
+ {
+ createDataSchema(false);
+ }
+ else
+ {
+ createDataSchema(fs, true);
+ }
+ DataSchemaLoader.initialize();
+
+ // Create and load the query schemas
+ // DNS_HOSTNAME_QUERY
+ List<String> dnsHostnameQueryElements = Arrays.asList(DATE, SRCIP, DSTIP, QTYPE, RCODE, IPS);
+ List<String> dnsHostnameQueryFilterElements = Collections.singletonList(QNAME);
+
+ TestUtils.createQuerySchema(DNS_HOSTNAME_QUERY_FILE, DNS_HOSTNAME_QUERY, TEST_DATA_SCHEMA_NAME, QNAME, dnsHostnameQueryElements,
+ dnsHostnameQueryFilterElements, filter);
+ if (hdfs)
+ {
+ TestUtils.createQuerySchema(DNS_HOSTNAME_QUERY_FILE_HDFS, DNS_HOSTNAME_QUERY, TEST_DATA_SCHEMA_NAME, QNAME, dnsHostnameQueryElements,
+ dnsHostnameQueryFilterElements, filter, false, fs, hdfs);
+ }
+
+ // DNS_IP_QUERY
+ List<String> dnsIPQueryElements = Arrays.asList(SRCIP, DSTIP, IPS);
+ List<String> dnsIPQueryFilterElements = Collections.singletonList(QNAME);
+
+ TestUtils.createQuerySchema(DNS_IP_QUERY_FILE, DNS_IP_QUERY, TEST_DATA_SCHEMA_NAME, IPS, dnsIPQueryElements, dnsIPQueryFilterElements, filter);
+ if (hdfs)
+ {
+ TestUtils.createQuerySchema(DNS_IP_QUERY_FILE_HDFS, DNS_IP_QUERY, TEST_DATA_SCHEMA_NAME, IPS, dnsIPQueryElements, dnsIPQueryFilterElements, filter,
+ false, fs, hdfs);
+ }
+
+ // DNS_NXDOMAIN_QUERY
+ List<String> dnsNXQueryElements = Arrays.asList(QNAME, SRCIP, DSTIP);
+ List<String> dnsNXQueryFilterElements = Collections.singletonList(QNAME);
+
+ TestUtils
+ .createQuerySchema(DNS_NXDOMAIN_QUERY_FILE, DNS_NXDOMAIN_QUERY, TEST_DATA_SCHEMA_NAME, QNAME, dnsNXQueryElements, dnsNXQueryFilterElements, filter);
+ if (hdfs)
+ {
+ TestUtils.createQuerySchema(DNS_NXDOMAIN_QUERY_FILE_HDFS, DNS_NXDOMAIN_QUERY, TEST_DATA_SCHEMA_NAME, QNAME, dnsNXQueryElements, dnsNXQueryFilterElements,
+ filter, false, fs, hdfs);
+ }
+
+ // DNS_SRCIP_QUERY
+ List<String> dnsSrcIPQueryElements = Arrays.asList(QNAME, DSTIP, IPS);
+ List<String> dnsSrcIPQueryFilterElements = Arrays.asList(SRCIP, IPS);
+
+ TestUtils
+ .createQuerySchema(DNS_SRCIP_QUERY_FILE, DNS_SRCIP_QUERY, TEST_DATA_SCHEMA_NAME, SRCIP, dnsSrcIPQueryElements, dnsSrcIPQueryFilterElements, filter);
+ if (hdfs)
+ {
+ TestUtils.createQuerySchema(DNS_SRCIP_QUERY_FILE_HDFS, DNS_SRCIP_QUERY, TEST_DATA_SCHEMA_NAME, SRCIP, dnsSrcIPQueryElements, dnsSrcIPQueryFilterElements,
+ filter, false, fs, hdfs);
+ }
+
+ // DNS_SRCIP_QUERY_NO_FILTER
+ List<String> dnsSrcIPQueryNoFilterElements = Arrays.asList(QNAME, DSTIP, IPS);
+ TestUtils.createQuerySchema(DNS_SRCIP_QUERY_NO_FILTER_FILE, DNS_SRCIP_QUERY_NO_FILTER, TEST_DATA_SCHEMA_NAME, SRCIP, dnsSrcIPQueryNoFilterElements, null,
+ null);
+ if (hdfs)
+ {
+ TestUtils.createQuerySchema(DNS_SRCIP_QUERY_NO_FILTER_FILE_HDFS, DNS_SRCIP_QUERY_NO_FILTER, TEST_DATA_SCHEMA_NAME, SRCIP, dnsSrcIPQueryNoFilterElements,
+ null, null, false, fs, hdfs);
+ }
+
+ QuerySchemaLoader.initialize();
+ }
+
+ /**
+ * Create the test data schema file
+ */
+ private static void createDataSchema(boolean hdfs) throws IOException
+ {
+ createDataSchema(null, hdfs);
+ }
+
+ /**
+ * Create the test data schema file
+ */
+ private static void createDataSchema(FileSystem fs, boolean hdfs) throws IOException
+ {
+ // Create a temporary file for the test schema, set in the properties
+ File file = File.createTempFile(DATA_SCHEMA_FILE_LOCALFS, ".xml");
+ file.deleteOnExit();
+ logger.info("file = " + file.toString());
+ SystemConfiguration.setProperty("data.schemas", file.toString());
+
+ // If we are performing distributed testing, write both the local and hdfs files
+ OutputStreamWriter osw = null;
+ if (hdfs)
+ {
+ Path filePath = new Path(DATA_SCHEMA_FILE_HDFS);
+ fs.deleteOnExit(filePath);
+ osw = new OutputStreamWriter(fs.create(filePath, true));
+
+ logger.info("hdfs: filePath = " + filePath.toString());
+ }
+
+ // Write to the file
+ try
+ {
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+ Document doc = dBuilder.newDocument();
+
+ // root element
+ Element rootElement = doc.createElement("schema");
+ doc.appendChild(rootElement);
+
+ // Add the schemaName
+ Element schemaNameElement = doc.createElement("schemaName");
+ schemaNameElement.appendChild(doc.createTextNode(TEST_DATA_SCHEMA_NAME));
+ rootElement.appendChild(schemaNameElement);
+
+ String primitiveTypePartitionerName = PrimitiveTypePartitioner.class.getName();
+ String ipPartitionerName = IPDataPartitioner.class.getName();
+ String datePartitioner = ISO8601DatePartitioner.class.getName();
+
+ // date
+ TestUtils.addElement(doc, rootElement, DATE, PrimitiveTypePartitioner.STRING, "false", datePartitioner);
+
+ // qname
+ TestUtils.addElement(doc, rootElement, QNAME, PrimitiveTypePartitioner.STRING, "false", primitiveTypePartitionerName);
+
+ // src_ip
+ TestUtils.addElement(doc, rootElement, SRCIP, PrimitiveTypePartitioner.STRING, "false", ipPartitionerName);
+
+ // dest_ip
+ TestUtils.addElement(doc, rootElement, DSTIP, PrimitiveTypePartitioner.STRING, "false", ipPartitionerName);
+
+ // qtype
+ TestUtils.addElement(doc, rootElement, QTYPE, PrimitiveTypePartitioner.SHORT, "true", primitiveTypePartitionerName);
+
+ // rcode
+ TestUtils.addElement(doc, rootElement, RCODE, PrimitiveTypePartitioner.INT, "false", primitiveTypePartitionerName);
+
+ // ip
+ TestUtils.addElement(doc, rootElement, IPS, PrimitiveTypePartitioner.STRING, "true", ipPartitionerName);
+
+ // Write to a xml file - both localFS and hdfs
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ Transformer transformer = transformerFactory.newTransformer();
+ DOMSource source = new DOMSource(doc);
+
+ // LocalFS
+ StreamResult resultLocalFS = new StreamResult(file);
+ transformer.transform(source, resultLocalFS);
+
+ if (hdfs)
+ {
+ StreamResult resultHDFS = new StreamResult(osw);
+ transformer.transform(source, resultHDFS);
+ }
+
+ // Output for testing
+ StreamResult consoleResult = new StreamResult(System.out);
+ transformer.transform(source, consoleResult);
+ System.out.println();
+
+ if (osw != null)
+ {
+ osw.close();
+ }
+
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/utils/StandaloneQuery.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/utils/StandaloneQuery.java b/src/test/java/org/apache/pirk/test/utils/StandaloneQuery.java
new file mode 100644
index 0000000..1c26bdd
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/utils/StandaloneQuery.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.utils;
+
+import org.apache.pirk.encryption.Paillier;
+import org.apache.pirk.querier.wideskies.Querier;
+import org.apache.pirk.querier.wideskies.QuerierConst;
+import org.apache.pirk.querier.wideskies.decrypt.DecryptResponse;
+import org.apache.pirk.querier.wideskies.encrypt.EncryptQuery;
+import org.apache.pirk.query.wideskies.Query;
+import org.apache.pirk.query.wideskies.QueryInfo;
+import org.apache.pirk.query.wideskies.QueryUtils;
+import org.apache.pirk.responder.wideskies.standalone.Responder;
+import org.apache.pirk.response.wideskies.Response;
+import org.apache.pirk.schema.query.QuerySchema;
+import org.apache.pirk.schema.query.QuerySchemaRegistry;
+import org.apache.pirk.schema.response.QueryResponseJSON;
+import org.apache.pirk.serialization.LocalFileSystemStore;
+import org.apache.pirk.utils.PIRException;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.junit.Assert.fail;
+
+public class StandaloneQuery
+{
+ private static final Logger logger = LoggerFactory.getLogger(StandaloneQuery.class);
+
+ static String queryFileDomain = "qfDomain";
+ static String queryFileIP = "qfIP";
+
+ String testDataSchemaName = "testDataSchema";
+ String testQuerySchemaName = "testQuerySchema";
+
+ // Base method to perform the query
+ public static List<QueryResponseJSON> performStandaloneQuery(List<JSONObject> dataElements, String queryType, List<String> selectors,
+ int numThreads, boolean testFalsePositive) throws IOException, InterruptedException, PIRException
+ {
+ logger.info("Performing watchlisting: ");
+
+ QuerySchema qSchema = QuerySchemaRegistry.get(queryType);
+
+ // Create the necessary files
+ LocalFileSystemStore storage = new LocalFileSystemStore();
+ String querySideOuputFilePrefix = "querySideOut";
+ File fileQuerier = File.createTempFile(querySideOuputFilePrefix + "-" + QuerierConst.QUERIER_FILETAG, ".txt");
+ File fileQuery = File.createTempFile(querySideOuputFilePrefix + "-" + QuerierConst.QUERY_FILETAG, ".txt");
+ String responseFile = "encryptedResponse";
+ File fileResponse = File.createTempFile(responseFile, ".txt");
+ String finalResultsFile = "finalResultFile";
+ File fileFinalResults = File.createTempFile(finalResultsFile, ".txt");
+
+ logger.info("fileQuerier = " + fileQuerier.getAbsolutePath() + " fileQuery = " + fileQuery.getAbsolutePath() + " responseFile = "
+ + fileResponse.getAbsolutePath() + " fileFinalResults = " + fileFinalResults.getAbsolutePath());
+
+ boolean embedSelector = SystemConfiguration.getBooleanProperty("pirTest.embedSelector", false);
+ boolean useExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useExpLookupTable", false);
+ boolean useHDFSExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useHDFSExpLookupTable", false);
+
+ // Set the necessary objects
+ QueryInfo queryInfo = new QueryInfo(BaseTests.queryIdentifier, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize,
+ queryType, useExpLookupTable, embedSelector, useHDFSExpLookupTable);
+
+ if (SystemConfiguration.getBooleanProperty("pir.embedQuerySchema", false))
+ {
+ queryInfo.addQuerySchema(qSchema);
+ }
+
+ Paillier paillier = new Paillier(BaseTests.paillierBitSize, BaseTests.certainty);
+
+ // Perform the encryption
+ logger.info("Performing encryption of the selectors - forming encrypted query vectors:");
+ EncryptQuery encryptQuery = new EncryptQuery(queryInfo, selectors, paillier);
+ encryptQuery.encrypt(numThreads);
+ logger.info("Completed encryption of the selectors - completed formation of the encrypted query vectors:");
+
+ // Dork with the embedSelectorMap to generate a false positive for the last valid selector in selectors
+ if (testFalsePositive)
+ {
+ Querier querier = encryptQuery.getQuerier();
+ HashMap<Integer,String> embedSelectorMap = querier.getEmbedSelectorMap();
+ logger.info("embedSelectorMap((embedSelectorMap.size()-2)) = " + embedSelectorMap.get((embedSelectorMap.size() - 2)) + " selector = "
+ + selectors.get((embedSelectorMap.size() - 2)));
+ embedSelectorMap.put((embedSelectorMap.size() - 2), "fakeEmbeddedSelector");
+ }
+
+ // Write necessary output files
+ storage.store(fileQuerier, encryptQuery.getQuerier());
+ storage.store(fileQuery, encryptQuery.getQuery());
+
+ // Perform the PIR query and build the response elements
+ logger.info("Performing the PIR Query and constructing the response elements:");
+ Query query = storage.recall(fileQuery, Query.class);
+ Responder pirResponder = new Responder(query);
+ logger.info("Query and Responder elements constructed");
+ for (JSONObject jsonData : dataElements)
+ {
+ String selector = QueryUtils.getSelectorByQueryTypeJSON(qSchema, jsonData);
+ logger.info("selector = " + selector + " numDataElements = " + jsonData.size());
+ try
+ {
+ pirResponder.addDataElement(selector, jsonData);
+ } catch (Exception e)
+ {
+ fail(e.toString());
+ }
+ }
+ logger.info("Completed the PIR Query and construction of the response elements:");
+
+ // Set the response object, extract, write to file
+ logger.info("Forming response from response elements; writing to a file");
+ pirResponder.setResponseElements();
+ Response responseOut = pirResponder.getResponse();
+ storage.store(fileResponse, responseOut);
+ logger.info("Completed forming response from response elements and writing to a file");
+
+ // Perform decryption
+ // Reconstruct the necessary objects from the files
+ logger.info("Performing decryption; writing final results file");
+ Response responseIn = storage.recall(fileResponse, Response.class);
+ Querier querier = storage.recall(fileQuerier, Querier.class);
+
+ // Perform decryption and output the result file
+ DecryptResponse decryptResponse = new DecryptResponse(responseIn, querier);
+ decryptResponse.decrypt(numThreads);
+ decryptResponse.writeResultFile(fileFinalResults);
+ logger.info("Completed performing decryption and writing final results file");
+
+ // Read in results
+ logger.info("Reading in and checking results");
+ List<QueryResponseJSON> results = TestUtils.readResultsFile(fileFinalResults);
+
+ // Clean up
+ fileQuerier.delete();
+ fileQuery.delete();
+ fileResponse.delete();
+ fileFinalResults.delete();
+
+ return results;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/utils/TestUtils.java b/src/test/java/org/apache/pirk/test/utils/TestUtils.java
new file mode 100644
index 0000000..1ea01fb
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/utils/TestUtils.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.utils;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.pirk.schema.response.QueryResponseJSON;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+/**
+ * Class to hold testing utilities
+ *
+ */
+public class TestUtils
+{
+ private static final Logger logger = LoggerFactory.getLogger(TestUtils.class);
+
+ /**
+ * Method to delete an ES index
+ */
+ public static void deleteESTestIndex(String index)
+ {
+ logger.info("Deleting index:");
+ ProcessBuilder pDelete = new ProcessBuilder("curl", "-XDELETE", index);
+ try
+ {
+ executeCommand(pDelete);
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Method to execute process
+ */
+ public static void executeCommand(ProcessBuilder p) throws IOException
+ {
+ Process proc = p.start();
+
+ try (BufferedReader stdInput = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+ BufferedReader stdError = new BufferedReader(new InputStreamReader(proc.getErrorStream())))
+ {
+ // Read the output from the command
+ logger.info("Standard output of the command:\n");
+ String s;
+ while ((s = stdInput.readLine()) != null)
+ {
+ logger.info(s);
+ }
+
+ // Read any errors from the attempted command
+ logger.info("Standard error of the command (if any):\n");
+ while ((s = stdError.readLine()) != null)
+ {
+ logger.info(s);
+ }
+ }
+ }
+
+ /**
+ * Helper method to add elements to the test data schema
+ */
+ public static void addElement(Document doc, Element rootElement, String elementName, String typeIn, String isArrayIn, String partitionerIn)
+ {
+ Element element = doc.createElement("element");
+ rootElement.appendChild(element);
+
+ Element name = doc.createElement("name");
+ name.appendChild(doc.createTextNode(elementName));
+ element.appendChild(name);
+
+ Element type = doc.createElement("type");
+ type.appendChild(doc.createTextNode(typeIn));
+ element.appendChild(type);
+
+ if (isArrayIn.equals("true"))
+ {
+ element.appendChild(doc.createElement("isArray"));
+ }
+
+ if (partitionerIn != null)
+ {
+ Element partitioner = doc.createElement("partitioner");
+ partitioner.appendChild(doc.createTextNode(partitionerIn));
+ element.appendChild(partitioner);
+ }
+ }
+
+ /**
+ * Creates the test query schema file
+ */
+ public static void createQuerySchema(String schemaFile, String querySchemaName, String dataSchemaNameInput, String selectorNameInput,
+ List<String> elementNames, List<String> filterNames, String filter) throws IOException
+ {
+ createQuerySchema(schemaFile, querySchemaName, dataSchemaNameInput, selectorNameInput, elementNames, filterNames, filter, true, null, false);
+ }
+
+ /**
+ * Creates the test query schema file
+ */
+ public static void createQuerySchema(String schemaFile, String querySchemaName, String dataSchemaNameInput, String selectorNameInput,
+ List<String> elementNames, List<String> filterNames, String filter, boolean append, FileSystem fs, boolean hdfs) throws IOException
+ {
+ logger.info("createQuerySchema: querySchemaName = " + querySchemaName);
+
+ // Create a temporary file for the test schema, set in the properties
+ String fileName;
+ File file = null;
+ OutputStreamWriter osw = null;
+ if (hdfs)
+ {
+ Path filePath = new Path(schemaFile);
+ fs.deleteOnExit(filePath);
+ fileName = filePath.toString();
+
+ osw = new OutputStreamWriter(fs.create(filePath, true));
+
+ logger.info("hdfs: filePath = " + fileName);
+ }
+ else
+ {
+ file = File.createTempFile(schemaFile, ".xml");
+ file.deleteOnExit();
+ fileName = file.toString();
+ logger.info("localFS: file = " + file.toString());
+ }
+
+ if (append)
+ {
+ String currentSchemas = SystemConfiguration.getProperty("query.schemas", "");
+ if (currentSchemas.equals("") || currentSchemas.equals("none"))
+ {
+ SystemConfiguration.setProperty("query.schemas", fileName);
+ }
+ else
+ {
+ SystemConfiguration.setProperty("query.schemas", SystemConfiguration.getProperty("query.schemas", "") + "," + fileName);
+ }
+ }
+ logger.info("query.schemas = " + SystemConfiguration.getProperty("query.schemas"));
+
+ // Write to the file
+ try
+ {
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+ Document doc = dBuilder.newDocument();
+
+ // root element
+ Element rootElement = doc.createElement("schema");
+ doc.appendChild(rootElement);
+
+ // Add the schemaName
+ Element schemaNameElement = doc.createElement("schemaName");
+ schemaNameElement.appendChild(doc.createTextNode(querySchemaName));
+ rootElement.appendChild(schemaNameElement);
+
+ // Add the dataSchemaName
+ Element dataSchemaNameElement = doc.createElement("dataSchemaName");
+ dataSchemaNameElement.appendChild(doc.createTextNode(dataSchemaNameInput));
+ rootElement.appendChild(dataSchemaNameElement);
+
+ // Add the selectorName
+ Element selectorNameElement = doc.createElement("selectorName");
+ selectorNameElement.appendChild(doc.createTextNode(selectorNameInput));
+ rootElement.appendChild(selectorNameElement);
+
+ // Add the elementNames
+ Element elements = doc.createElement("elements");
+ rootElement.appendChild(elements);
+ for (String elementName : elementNames)
+ {
+ logger.info("elementName = " + elementName);
+ Element name = doc.createElement("name");
+ name.appendChild(doc.createTextNode(elementName));
+ elements.appendChild(name);
+ }
+
+ // Add the filter
+ if (filter != null)
+ {
+ Element filterElement = doc.createElement("filter");
+ filterElement.appendChild(doc.createTextNode(filter));
+ rootElement.appendChild(filterElement);
+
+ // Add the filterNames
+ Element filterNamesElement = doc.createElement("filterNames");
+ rootElement.appendChild(filterNamesElement);
+ for (String filterName : filterNames)
+ {
+ logger.info("filterName = " + filterName);
+ Element name = doc.createElement("name");
+ name.appendChild(doc.createTextNode(filterName));
+ filterNamesElement.appendChild(name);
+ }
+ }
+
+ // Write to a xml file
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ Transformer transformer = transformerFactory.newTransformer();
+ DOMSource source = new DOMSource(doc);
+ StreamResult result;
+ if (hdfs)
+ {
+ result = new StreamResult(osw);
+ }
+ else
+ {
+ result = new StreamResult(file);
+ }
+ transformer.transform(source, result);
+
+ // Output for testing
+ StreamResult consoleResult = new StreamResult(System.out);
+ transformer.transform(source, consoleResult);
+ System.out.println();
+
+ if (osw != null)
+ {
+ osw.close();
+ }
+
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Converts the result file into an ArrayList of QueryResponseJSON objects
+ */
+ public static List<QueryResponseJSON> readResultsFile(File file)
+ {
+ List<QueryResponseJSON> results = new ArrayList<>();
+ try (BufferedReader br = new BufferedReader(new FileReader(file)))
+ {
+ String line;
+ while ((line = br.readLine()) != null)
+ {
+ QueryResponseJSON jsonResult = new QueryResponseJSON(line);
+ results.add(jsonResult);
+ }
+ } catch (Exception e)
+ {
+ logger.error(e.toString());
+ }
+
+ return results;
+ }
+
+ /**
+ * Write the ArrayList<String to a tmp file in the local filesystem with the given fileName
+ *
+ */
+ public static String writeToTmpFile(List<String> list, String fileName, String suffix) throws IOException
+ {
+ File file = File.createTempFile(fileName, suffix);
+ file.deleteOnExit();
+ logger.info("localFS: file = " + file);
+
+ FileWriter fw = new FileWriter(file);
+ try (BufferedWriter bw = new BufferedWriter(fw))
+ {
+ for (String s : list)
+ {
+ bw.write(s);
+ bw.newLine();
+ }
+ }
+
+ return file.getPath();
+ }
+}
[5/6] incubator-pirk git commit: Move all tests under src/test
directory, this closes apache/incubator-pirk#66
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/main/java/org/apache/pirk/test/utils/Inputs.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/test/utils/Inputs.java b/src/main/java/org/apache/pirk/test/utils/Inputs.java
deleted file mode 100644
index 10c1386..0000000
--- a/src/main/java/org/apache/pirk/test/utils/Inputs.java
+++ /dev/null
@@ -1,606 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.test.utils;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.pirk.schema.data.DataSchemaLoader;
-import org.apache.pirk.schema.data.partitioner.IPDataPartitioner;
-import org.apache.pirk.schema.data.partitioner.ISO8601DatePartitioner;
-import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner;
-import org.apache.pirk.schema.query.QuerySchemaLoader;
-import org.apache.pirk.test.distributed.DistributedTestDriver;
-import org.apache.pirk.utils.HDFS;
-import org.apache.pirk.utils.PIRException;
-import org.apache.pirk.utils.SystemConfiguration;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-
-/**
- * Input files for distributed testing
- *
- */
-public class Inputs
-{
- private static final Logger logger = LoggerFactory.getLogger(Inputs.class);
-
- // Test data schema fields
- public static final String DATE = "date";
- public static final String QNAME = "qname";
- public static final String SRCIP = "src_ip";
- public static final String DSTIP = "dest_ip";
- public static final String QTYPE = "qtype";
- public static final String RCODE = "rcode";
- public static final String IPS = "ip";
-
- // Test query types
- public static final String DNS_HOSTNAME_QUERY = "dns-hostname-query"; // Query for the watched hostnames occurred; ; watched value type -- hostname
- public static final String DNS_IP_QUERY = "dns-ip-query"; // The watched IP address(es) were detected in the response to a query; watched value type -- IP in
- // IPS field (resolution IP)
- public static final String DNS_NXDOMAIN_QUERY = "dns-nxdomain-query"; // Query for nxdomain responses that were made for watched qnames
- public static final String DNS_SRCIP_QUERY = "dns-srcip-query"; // Query for responses from watched srcIPs
- public static final String DNS_SRCIP_QUERY_NO_FILTER = "dns-srcip-query-no-filter"; // Query for responses from watched srcIPs, no data filter used
-
- // Test query type files - localfs
- public static final String DNS_HOSTNAME_QUERY_FILE = DNS_HOSTNAME_QUERY + "_file";
- public static final String DNS_IP_QUERY_FILE = DNS_IP_QUERY + "_file";
- public static final String DNS_NXDOMAIN_QUERY_FILE = DNS_NXDOMAIN_QUERY + "_file";
- public static final String DNS_SRCIP_QUERY_FILE = DNS_SRCIP_QUERY + "_file";
- public static final String DNS_SRCIP_QUERY_NO_FILTER_FILE = DNS_SRCIP_QUERY_NO_FILTER + "_file";
-
- // Test query files hdfs
- public static final String DNS_HOSTNAME_QUERY_FILE_HDFS = "/tmp/" + DNS_HOSTNAME_QUERY + "_file";
- public static final String DNS_IP_QUERY_FILE_HDFS = "/tmp/" + DNS_IP_QUERY + "_file";
- public static final String DNS_NXDOMAIN_QUERY_FILE_HDFS = "/tmp/" + DNS_NXDOMAIN_QUERY + "_file";
- public static final String DNS_SRCIP_QUERY_FILE_HDFS = "/tmp/" + DNS_SRCIP_QUERY + "_file";
- public static final String DNS_SRCIP_QUERY_NO_FILTER_FILE_HDFS = "/tmp/" + DNS_SRCIP_QUERY_NO_FILTER + "_file";
-
- // Combined query file strings -- used to set properties
- public static final String LOCALFS_QUERY_FILES = DNS_HOSTNAME_QUERY_FILE + "," + DNS_IP_QUERY_FILE + "," + DNS_NXDOMAIN_QUERY_FILE + ","
- + DNS_SRCIP_QUERY_FILE + "," + DNS_SRCIP_QUERY_NO_FILTER_FILE;
-
- public static final String HDFS_QUERY_FILES = DNS_HOSTNAME_QUERY_FILE_HDFS + "," + DNS_IP_QUERY_FILE_HDFS + "," + DNS_NXDOMAIN_QUERY_FILE_HDFS + ","
- + DNS_SRCIP_QUERY_FILE_HDFS + "," + DNS_SRCIP_QUERY_NO_FILTER_FILE_HDFS;
-
- // Test data schema files -- localFS and hdfs
- public static final String TEST_DATA_SCHEMA_NAME = "testDataSchema";
- public static final String DATA_SCHEMA_FILE_LOCALFS = "testDataSchemaFile";
- public static final String DATA_SCHEMA_FILE_HDFS = "/tmp/testDataSchemaFile.xml";
-
- /**
- * Delete the ElasticSearch indices that was used for functional testing
- */
- public static void deleteESInput()
- {
- String esPIRIndex = SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_NODES_PROPERTY) + ":"
- + SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_PORT_PROPERTY) + "/"
- + SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_INDEX_PROPERTY);
- logger.info("ES input being deleted at " + esPIRIndex);
-
- ProcessBuilder pDeletePIR = new ProcessBuilder("curl", "-XDELETE", esPIRIndex);
- try
- {
- TestUtils.executeCommand(pDeletePIR);
- logger.info("ES input deleted!");
- } catch (IOException e)
- {
- e.printStackTrace();
- }
- }
-
- /**
- * Creates PIR JSON input
- */
- @SuppressWarnings("unchecked")
- public static ArrayList<JSONObject> createJSONDataElements()
- {
- ArrayList<JSONObject> dataElementsJSON = new ArrayList<>();
-
- JSONObject jsonObj1 = new JSONObject();
- jsonObj1.put(DATE, "2016-02-20T23:29:05.000Z");
- jsonObj1.put(QNAME, "a.b.c.com"); // hits on domain selector
- jsonObj1.put(SRCIP, "55.55.55.55"); // hits on IP selector
- jsonObj1.put(DSTIP, "1.2.3.6");
- jsonObj1.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
- jsonObj1.put(RCODE, 0);
- jsonObj1.put(IPS, new ArrayList<>(Arrays.asList("10.20.30.40", "10.20.30.60")));
-
- dataElementsJSON.add(jsonObj1);
-
- JSONObject jsonObj2 = new JSONObject();
- jsonObj2.put(DATE, "2016-02-20T23:29:06.000Z");
- jsonObj2.put(QNAME, "d.e.com");
- jsonObj2.put(SRCIP, "127.128.129.130");
- jsonObj2.put(DSTIP, "1.2.3.4");
- jsonObj2.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
- jsonObj2.put(RCODE, 0);
- jsonObj2.put(IPS, new ArrayList<>(Collections.singletonList("5.6.7.8")));
-
- dataElementsJSON.add(jsonObj2);
-
- JSONObject jsonObj3 = new JSONObject();
- jsonObj3.put(DATE, "2016-02-20T23:29:07.000Z");
- jsonObj3.put(QNAME, "d.e.com");
- jsonObj3.put(SRCIP, "131.132.133.134");
- jsonObj3.put(DSTIP, "9.10.11.12");
- jsonObj3.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
- jsonObj3.put(RCODE, 0);
- jsonObj3.put(IPS, new ArrayList<>(Collections.singletonList("13.14.15.16")));
-
- dataElementsJSON.add(jsonObj3);
-
- JSONObject jsonObj4 = new JSONObject();
- jsonObj4.put(DATE, "2016-02-20T23:29:08.000Z");
- jsonObj4.put(QNAME, "d.e.com");
- jsonObj4.put(SRCIP, "135.136.137.138");
- jsonObj4.put(DSTIP, "17.18.19.20");
- jsonObj4.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
- jsonObj4.put(RCODE, 3);
- jsonObj4.put(IPS, new ArrayList<>(Collections.singletonList("21.22.23.24")));
-
- dataElementsJSON.add(jsonObj4);
-
- JSONObject jsonObj5 = new JSONObject();
- jsonObj5.put(DATE, "2016-02-20T23:29:09.000Z");
- jsonObj5.put(QNAME, "d.e.com");
- jsonObj5.put(SRCIP, "139.140.141.142");
- jsonObj5.put(DSTIP, "25.26.27.28");
- jsonObj5.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
- jsonObj5.put(RCODE, 0);
- jsonObj5.put(IPS, new ArrayList<>(Collections.singletonList("5.6.7.8")));
-
- dataElementsJSON.add(jsonObj5);
-
- JSONObject jsonObj6 = new JSONObject();
- jsonObj6.put(DATE, "2016-02-20T23:29:10.000Z");
- jsonObj6.put(QNAME, "d.e.com");
- jsonObj6.put(SRCIP, "143.144.145.146");
- jsonObj6.put(DSTIP, "33.34.35.36");
- jsonObj6.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
- jsonObj6.put(RCODE, 0);
- jsonObj6.put(IPS, new ArrayList<>(Collections.singletonList("5.6.7.8")));
-
- dataElementsJSON.add(jsonObj6);
-
- JSONObject jsonObj7 = new JSONObject();
- jsonObj7.put(DATE, "2016-02-20T23:29:11.000Z");
- jsonObj7.put(QNAME, "something.else");
- jsonObj7.put(SRCIP, "1.1.1.1");
- jsonObj7.put(DSTIP, "2.2.2.2");
- jsonObj7.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
- jsonObj7.put(RCODE, 0);
- jsonObj7.put(IPS, new ArrayList<>(Collections.singletonList("3.3.3.3")));
-
- dataElementsJSON.add(jsonObj7);
-
- // This should never be returned - doesn't hit on any domain selectors
- // resolution ip on stoplist
- JSONObject jsonObj8 = new JSONObject();
- jsonObj8.put(DATE, "2016-02-20T23:29:12.000Z");
- jsonObj8.put(QNAME, "something.else2");
- jsonObj8.put(SRCIP, "5.6.7.8");
- jsonObj8.put(DSTIP, "2.2.2.22");
- jsonObj8.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
- jsonObj8.put(RCODE, 0);
- jsonObj8.put(IPS, new ArrayList<>(Collections.singletonList("3.3.3.132")));
-
- dataElementsJSON.add(jsonObj8);
-
- // This should never be returned in distributed case -- domain and resolution ip on stoplist
- JSONObject jsonObj9 = new JSONObject();
- jsonObj9.put(DATE, "2016-02-20T23:29:13.000Z");
- jsonObj9.put(QNAME, "something.else.on.stoplist");
- jsonObj9.put(SRCIP, "55.55.55.55");
- jsonObj9.put(DSTIP, "2.2.2.232");
- jsonObj9.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
- jsonObj9.put(RCODE, 0);
- jsonObj9.put(IPS, new ArrayList<>(Collections.singletonList("3.3.3.132")));
-
- dataElementsJSON.add(jsonObj9);
-
- return dataElementsJSON;
- }
-
- /**
- * Creates an ArrayList of JSONObjects with RCODE value of 3
- */
- @SuppressWarnings("unchecked")
- public static ArrayList<JSONObject> getRcode3JSONDataElements()
- {
- ArrayList<JSONObject> dataElementsJSON = new ArrayList<>();
-
- JSONObject jsonObj4 = new JSONObject();
- jsonObj4.put(DATE, "2016-02-20T23:29:08.000Z");
- jsonObj4.put(QNAME, "d.e.com");
- jsonObj4.put(SRCIP, "135.136.137.138");
- jsonObj4.put(DSTIP, "17.18.19.20");
- jsonObj4.put(QTYPE, new ArrayList<>(Collections.singletonList((short) 1)));
- jsonObj4.put(RCODE, 3);
- jsonObj4.put(IPS, new ArrayList<>(Collections.singletonList("21.22.23.24")));
-
- dataElementsJSON.add(jsonObj4);
-
- return dataElementsJSON;
- }
-
- /**
- * Creates PIR JSON input and writes to hdfs
- */
- public static List<JSONObject> createPIRJSONInput(FileSystem fs)
- {
- String inputJSONFile = SystemConfiguration.getProperty(DistributedTestDriver.JSON_PIR_INPUT_FILE_PROPERTY);
- logger.info("PIR JSON input being created at " + inputJSONFile);
-
- List<JSONObject> dataElementsJSON = createJSONDataElements();
-
- HDFS.writeFile(dataElementsJSON, fs, inputJSONFile, true);
- logger.info("PIR JSON input successfully created!");
-
- return dataElementsJSON;
- }
-
- /**
- * Creates PIR Elasticsearch input
- */
- public static void createPIRESInput()
- {
- String esTestIndex = SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_NODES_PROPERTY) + ":"
- + SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_PORT_PROPERTY) + "/"
- + SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_INDEX_PROPERTY);
- String esType = SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_TYPE_PROPERTY);
- logger.info("ES input being created at " + esTestIndex + " with type " + esType);
-
- // Create ES Index
- logger.info("Creating new testindex:");
- ProcessBuilder pCreate = new ProcessBuilder("curl", "-XPUT", esTestIndex);
- try
- {
- TestUtils.executeCommand(pCreate);
- } catch (IOException e)
- {
- e.printStackTrace();
- }
-
- // Add elements
- logger.info(" \n \n Adding elements to testindex:");
-
- String indexTypeNum1 = esTestIndex + "/" + esType + "/1";
- logger.info("indexTypeNum1 = " + indexTypeNum1);
- ProcessBuilder pAdd1 = new ProcessBuilder("curl", "-XPUT", indexTypeNum1, "-d",
- "{\"qname\":\"a.b.c.com\",\"date\":\"2016-02-20T23:29:05.000Z\",\"qtype\":[\"1\"]"
- + ",\"rcode\":\"0\",\"src_ip\":\"55.55.55.55\",\"dest_ip\":\"1.2.3.6\"" + ",\"ip\":[\"10.20.30.40\",\"10.20.30.60\"]}");
-
- String indexTypeNum2 = esTestIndex + "/" + esType + "/2";
- logger.info("indexTypeNum2 = " + indexTypeNum2);
- ProcessBuilder pAdd2 = new ProcessBuilder("curl", "-XPUT", indexTypeNum2, "-d",
- "{\"qname\":\"d.e.com\",\"date\":\"2016-02-20T23:29:06.000Z\",\"qtype\":[\"1\"]"
- + ",\"rcode\":\"0\",\"src_ip\":\"127.128.129.130\",\"dest_ip\":\"1.2.3.4\"" + ",\"ip\":[\"5.6.7.8\"]}");
-
- String indexTypeNum3 = esTestIndex + "/" + esType + "/3";
- logger.info("indexTypeNum3 = " + indexTypeNum3);
- ProcessBuilder pAdd3 = new ProcessBuilder("curl", "-XPUT", indexTypeNum3, "-d",
- "{\"qname\":\"d.e.com\",\"date\":\"2016-02-20T23:29:07.000Z\",\"qtype\":[\"1\"]"
- + ",\"rcode\":\"0\",\"src_ip\":\"131.132.133.134\",\"dest_ip\":\"9.10.11.12\"" + ",\"ip\":[\"13.14.15.16\"]}");
-
- String indexTypeNum4 = esTestIndex + "/" + esType + "/4";
- logger.info("indexTypeNum4 = " + indexTypeNum4);
- ProcessBuilder pAdd4 = new ProcessBuilder("curl", "-XPUT", indexTypeNum4, "-d",
- "{\"qname\":\"d.e.com\",\"date\":\"2016-02-20T23:29:08.000Z\",\"qtype\":[\"1\"]"
- + ",\"rcode\":\"3\",\"src_ip\":\"135.136.137.138\",\"dest_ip\":\"17.18.19.20\"" + ",\"ip\":[\"21.22.23.24\"]}");
-
- String indexTypeNum5 = esTestIndex + "/" + esType + "/5";
- logger.info("indexTypeNum5 = " + indexTypeNum5);
- ProcessBuilder pAdd5 = new ProcessBuilder("curl", "-XPUT", indexTypeNum5, "-d",
- "{\"qname\":\"d.e.com\",\"date\":\"2016-02-20T23:29:09.000Z\",\"qtype\":[\"1\"]"
- + ",\"rcode\":\"0\",\"src_ip\":\"139.140.141.142\",\"dest_ip\":\"25.26.27.28\"" + ",\"ip\":[\"5.6.7.8\"]}");
-
- String indexTypeNum6 = esTestIndex + "/" + esType + "/6";
- logger.info("indexTypeNum6 = " + indexTypeNum6);
- ProcessBuilder pAdd6 = new ProcessBuilder("curl", "-XPUT", indexTypeNum6, "-d",
- "{\"qname\":\"d.e.com\",\"date\":\"2016-02-20T23:29:10.000Z\",\"qtype\":[\"1\"]"
- + ",\"rcode\":\"0\",\"src_ip\":\"143.144.145.146\",\"dest_ip\":\"33.34.35.36\"" + ",\"ip\":[\"5.6.7.8\"]}");
-
- String indexTypeNum7 = esTestIndex + "/" + esType + "/7";
- logger.info("indexTypeNum7 = " + indexTypeNum7);
- ProcessBuilder pAdd7 = new ProcessBuilder("curl", "-XPUT", indexTypeNum7, "-d",
- "{\"qname\":\"something.else\",\"date\":\"2016-02-20T23:29:11.000Z\",\"qtype\":[\"1\"]"
- + ",\"rcode\":\"0\",\"src_ip\":\"1.1.1.1\",\"dest_ip\":\"2.2.2.2\"" + ",\"ip\":[\"3.3.3.3\"]}");
-
- // Never should be returned - doesn't hit on any selectors
- String indexTypeNum8 = esTestIndex + "/" + esType + "/8";
- logger.info("indexTypeNum8 = " + indexTypeNum8);
- ProcessBuilder pAdd8 = new ProcessBuilder("curl", "-XPUT", indexTypeNum8, "-d",
- "{\"qname\":\"something.else2\",\"date\":\"2016-02-20T23:29:12.000Z\",\"qtype\":[\"1\"]"
- + ",\"rcode\":\"0\",\"src_ip\":\"1.1.1.12\",\"dest_ip\":\"2.2.2.22\"" + ",\"ip\":[\"3.3.3.32\"]}");
-
- // This should never be returned -- domain on stoplist
- String indexTypeNum9 = esTestIndex + "/" + esType + "/9";
- logger.info("indexTypeNum9 = " + indexTypeNum9);
- ProcessBuilder pAdd9 = new ProcessBuilder("curl", "-XPUT", indexTypeNum9, "-d",
- "{\"qname\":\"something.else.on.stoplist\",\"date\":\"2016-02-20T23:29:13.000Z\",\"qtype\":[\"1\"]"
- + ",\"rcode\":\"0\",\"src_ip\":\"55.55.55.55\",\"dest_ip\":\"2.2.2.232\"" + ",\"ip\":[\"3.3.3.132\"]}");
-
- try
- {
- TestUtils.executeCommand(pAdd1);
- TestUtils.executeCommand(pAdd2);
- TestUtils.executeCommand(pAdd3);
- TestUtils.executeCommand(pAdd4);
- TestUtils.executeCommand(pAdd5);
- TestUtils.executeCommand(pAdd6);
- TestUtils.executeCommand(pAdd7);
- TestUtils.executeCommand(pAdd8);
- TestUtils.executeCommand(pAdd9);
- } catch (IOException e)
- {
- e.printStackTrace();
- }
-
- // Retrieve and print all of the elements
- for (int i = 1; i < 7; ++i)
- {
- logger.info("Retrieving element number = " + i + " from " + esTestIndex);
- String elementGet = esTestIndex + "/" + esType + "/" + i;
- logger.info("elementGet = " + elementGet);
- ProcessBuilder pGet = new ProcessBuilder("curl", "-XGET", elementGet);
- try
- {
- TestUtils.executeCommand(pGet);
- } catch (IOException e)
- {
- e.printStackTrace();
- }
- }
- }
-
- /**
- * Creates PIR stoplist file
- */
- public static String createPIRStopList(FileSystem fs, boolean hdfs) throws IOException, PIRException
- {
- logger.info("PIR stopList file being created");
-
- List<String> elements = Arrays.asList("something.else.on.stoplist", "3.3.3.132");
-
- if (hdfs)
- {
- String pirStopListFile = SystemConfiguration.getProperty(DistributedTestDriver.PIR_STOPLIST_FILE);
- if (pirStopListFile == null)
- {
- throw new PIRException("HDFS stop list file configuration name is required.");
- }
- HDFS.writeFile(elements, fs, pirStopListFile, true);
- logger.info("pirStopListFile file successfully created on hdfs!");
- }
-
- String prefix = SystemConfiguration.getProperty("pir.stopListFile");
- if (prefix == null)
- {
- throw new PIRException("Local stop list file configuration name is required.");
- }
- return TestUtils.writeToTmpFile(elements, prefix, null);
- }
-
- /**
- * Create and load the data and query schema files used for testing
- */
- public static void createSchemaFiles(String filter) throws Exception
- {
- createSchemaFiles(null, false, filter);
- }
-
- /**
- * Create and load the data and query schema files used for testing
- * <p>
- * Writes both local and hdfs schema files if hdfs=true -- only updates the corresponding properties for the local files
- */
- public static void createSchemaFiles(FileSystem fs, boolean hdfs, String filter) throws Exception
- {
- // Create and load the data schema
- if (!hdfs)
- {
- createDataSchema(false);
- }
- else
- {
- createDataSchema(fs, true);
- }
- DataSchemaLoader.initialize();
-
- // Create and load the query schemas
- // DNS_HOSTNAME_QUERY
- List<String> dnsHostnameQueryElements = Arrays.asList(DATE, SRCIP, DSTIP, QTYPE, RCODE, IPS);
- List<String> dnsHostnameQueryFilterElements = Collections.singletonList(QNAME);
-
- TestUtils.createQuerySchema(DNS_HOSTNAME_QUERY_FILE, DNS_HOSTNAME_QUERY, TEST_DATA_SCHEMA_NAME, QNAME, dnsHostnameQueryElements,
- dnsHostnameQueryFilterElements, filter);
- if (hdfs)
- {
- TestUtils.createQuerySchema(DNS_HOSTNAME_QUERY_FILE_HDFS, DNS_HOSTNAME_QUERY, TEST_DATA_SCHEMA_NAME, QNAME, dnsHostnameQueryElements,
- dnsHostnameQueryFilterElements, filter, false, fs, hdfs);
- }
-
- // DNS_IP_QUERY
- List<String> dnsIPQueryElements = Arrays.asList(SRCIP, DSTIP, IPS);
- List<String> dnsIPQueryFilterElements = Collections.singletonList(QNAME);
-
- TestUtils.createQuerySchema(DNS_IP_QUERY_FILE, DNS_IP_QUERY, TEST_DATA_SCHEMA_NAME, IPS, dnsIPQueryElements, dnsIPQueryFilterElements, filter);
- if (hdfs)
- {
- TestUtils.createQuerySchema(DNS_IP_QUERY_FILE_HDFS, DNS_IP_QUERY, TEST_DATA_SCHEMA_NAME, IPS, dnsIPQueryElements, dnsIPQueryFilterElements, filter,
- false, fs, hdfs);
- }
-
- // DNS_NXDOMAIN_QUERY
- List<String> dnsNXQueryElements = Arrays.asList(QNAME, SRCIP, DSTIP);
- List<String> dnsNXQueryFilterElements = Collections.singletonList(QNAME);
-
- TestUtils
- .createQuerySchema(DNS_NXDOMAIN_QUERY_FILE, DNS_NXDOMAIN_QUERY, TEST_DATA_SCHEMA_NAME, QNAME, dnsNXQueryElements, dnsNXQueryFilterElements, filter);
- if (hdfs)
- {
- TestUtils.createQuerySchema(DNS_NXDOMAIN_QUERY_FILE_HDFS, DNS_NXDOMAIN_QUERY, TEST_DATA_SCHEMA_NAME, QNAME, dnsNXQueryElements, dnsNXQueryFilterElements,
- filter, false, fs, hdfs);
- }
-
- // DNS_SRCIP_QUERY
- List<String> dnsSrcIPQueryElements = Arrays.asList(QNAME, DSTIP, IPS);
- List<String> dnsSrcIPQueryFilterElements = Arrays.asList(SRCIP, IPS);
-
- TestUtils
- .createQuerySchema(DNS_SRCIP_QUERY_FILE, DNS_SRCIP_QUERY, TEST_DATA_SCHEMA_NAME, SRCIP, dnsSrcIPQueryElements, dnsSrcIPQueryFilterElements, filter);
- if (hdfs)
- {
- TestUtils.createQuerySchema(DNS_SRCIP_QUERY_FILE_HDFS, DNS_SRCIP_QUERY, TEST_DATA_SCHEMA_NAME, SRCIP, dnsSrcIPQueryElements, dnsSrcIPQueryFilterElements,
- filter, false, fs, hdfs);
- }
-
- // DNS_SRCIP_QUERY_NO_FILTER
- List<String> dnsSrcIPQueryNoFilterElements = Arrays.asList(QNAME, DSTIP, IPS);
- TestUtils.createQuerySchema(DNS_SRCIP_QUERY_NO_FILTER_FILE, DNS_SRCIP_QUERY_NO_FILTER, TEST_DATA_SCHEMA_NAME, SRCIP, dnsSrcIPQueryNoFilterElements, null,
- null);
- if (hdfs)
- {
- TestUtils.createQuerySchema(DNS_SRCIP_QUERY_NO_FILTER_FILE_HDFS, DNS_SRCIP_QUERY_NO_FILTER, TEST_DATA_SCHEMA_NAME, SRCIP, dnsSrcIPQueryNoFilterElements,
- null, null, false, fs, hdfs);
- }
-
- QuerySchemaLoader.initialize();
- }
-
- /**
- * Create the test data schema file
- */
- private static void createDataSchema(boolean hdfs) throws IOException
- {
- createDataSchema(null, hdfs);
- }
-
- /**
- * Create the test data schema file
- */
- private static void createDataSchema(FileSystem fs, boolean hdfs) throws IOException
- {
- // Create a temporary file for the test schema, set in the properties
- File file = File.createTempFile(DATA_SCHEMA_FILE_LOCALFS, ".xml");
- file.deleteOnExit();
- logger.info("file = " + file.toString());
- SystemConfiguration.setProperty("data.schemas", file.toString());
-
- // If we are performing distributed testing, write both the local and hdfs files
- OutputStreamWriter osw = null;
- if (hdfs)
- {
- Path filePath = new Path(DATA_SCHEMA_FILE_HDFS);
- fs.deleteOnExit(filePath);
- osw = new OutputStreamWriter(fs.create(filePath, true));
-
- logger.info("hdfs: filePath = " + filePath.toString());
- }
-
- // Write to the file
- try
- {
- DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
- DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
- Document doc = dBuilder.newDocument();
-
- // root element
- Element rootElement = doc.createElement("schema");
- doc.appendChild(rootElement);
-
- // Add the schemaName
- Element schemaNameElement = doc.createElement("schemaName");
- schemaNameElement.appendChild(doc.createTextNode(TEST_DATA_SCHEMA_NAME));
- rootElement.appendChild(schemaNameElement);
-
- String primitiveTypePartitionerName = PrimitiveTypePartitioner.class.getName();
- String ipPartitionerName = IPDataPartitioner.class.getName();
- String datePartitioner = ISO8601DatePartitioner.class.getName();
-
- // date
- TestUtils.addElement(doc, rootElement, DATE, PrimitiveTypePartitioner.STRING, "false", datePartitioner);
-
- // qname
- TestUtils.addElement(doc, rootElement, QNAME, PrimitiveTypePartitioner.STRING, "false", primitiveTypePartitionerName);
-
- // src_ip
- TestUtils.addElement(doc, rootElement, SRCIP, PrimitiveTypePartitioner.STRING, "false", ipPartitionerName);
-
- // dest_ip
- TestUtils.addElement(doc, rootElement, DSTIP, PrimitiveTypePartitioner.STRING, "false", ipPartitionerName);
-
- // qtype
- TestUtils.addElement(doc, rootElement, QTYPE, PrimitiveTypePartitioner.SHORT, "true", primitiveTypePartitionerName);
-
- // rcode
- TestUtils.addElement(doc, rootElement, RCODE, PrimitiveTypePartitioner.INT, "false", primitiveTypePartitionerName);
-
- // ip
- TestUtils.addElement(doc, rootElement, IPS, PrimitiveTypePartitioner.STRING, "true", ipPartitionerName);
-
- // Write to a xml file - both localFS and hdfs
- TransformerFactory transformerFactory = TransformerFactory.newInstance();
- Transformer transformer = transformerFactory.newTransformer();
- DOMSource source = new DOMSource(doc);
-
- // LocalFS
- StreamResult resultLocalFS = new StreamResult(file);
- transformer.transform(source, resultLocalFS);
-
- if (hdfs)
- {
- StreamResult resultHDFS = new StreamResult(osw);
- transformer.transform(source, resultHDFS);
- }
-
- // Output for testing
- StreamResult consoleResult = new StreamResult(System.out);
- transformer.transform(source, consoleResult);
- System.out.println();
-
- if (osw != null)
- {
- osw.close();
- }
-
- } catch (Exception e)
- {
- e.printStackTrace();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java b/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java
deleted file mode 100644
index 1c26bdd..0000000
--- a/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.test.utils;
-
-import org.apache.pirk.encryption.Paillier;
-import org.apache.pirk.querier.wideskies.Querier;
-import org.apache.pirk.querier.wideskies.QuerierConst;
-import org.apache.pirk.querier.wideskies.decrypt.DecryptResponse;
-import org.apache.pirk.querier.wideskies.encrypt.EncryptQuery;
-import org.apache.pirk.query.wideskies.Query;
-import org.apache.pirk.query.wideskies.QueryInfo;
-import org.apache.pirk.query.wideskies.QueryUtils;
-import org.apache.pirk.responder.wideskies.standalone.Responder;
-import org.apache.pirk.response.wideskies.Response;
-import org.apache.pirk.schema.query.QuerySchema;
-import org.apache.pirk.schema.query.QuerySchemaRegistry;
-import org.apache.pirk.schema.response.QueryResponseJSON;
-import org.apache.pirk.serialization.LocalFileSystemStore;
-import org.apache.pirk.utils.PIRException;
-import org.apache.pirk.utils.SystemConfiguration;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-
-import static org.junit.Assert.fail;
-
-public class StandaloneQuery
-{
- private static final Logger logger = LoggerFactory.getLogger(StandaloneQuery.class);
-
- static String queryFileDomain = "qfDomain";
- static String queryFileIP = "qfIP";
-
- String testDataSchemaName = "testDataSchema";
- String testQuerySchemaName = "testQuerySchema";
-
- // Base method to perform the query
- public static List<QueryResponseJSON> performStandaloneQuery(List<JSONObject> dataElements, String queryType, List<String> selectors,
- int numThreads, boolean testFalsePositive) throws IOException, InterruptedException, PIRException
- {
- logger.info("Performing watchlisting: ");
-
- QuerySchema qSchema = QuerySchemaRegistry.get(queryType);
-
- // Create the necessary files
- LocalFileSystemStore storage = new LocalFileSystemStore();
- String querySideOuputFilePrefix = "querySideOut";
- File fileQuerier = File.createTempFile(querySideOuputFilePrefix + "-" + QuerierConst.QUERIER_FILETAG, ".txt");
- File fileQuery = File.createTempFile(querySideOuputFilePrefix + "-" + QuerierConst.QUERY_FILETAG, ".txt");
- String responseFile = "encryptedResponse";
- File fileResponse = File.createTempFile(responseFile, ".txt");
- String finalResultsFile = "finalResultFile";
- File fileFinalResults = File.createTempFile(finalResultsFile, ".txt");
-
- logger.info("fileQuerier = " + fileQuerier.getAbsolutePath() + " fileQuery = " + fileQuery.getAbsolutePath() + " responseFile = "
- + fileResponse.getAbsolutePath() + " fileFinalResults = " + fileFinalResults.getAbsolutePath());
-
- boolean embedSelector = SystemConfiguration.getBooleanProperty("pirTest.embedSelector", false);
- boolean useExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useExpLookupTable", false);
- boolean useHDFSExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useHDFSExpLookupTable", false);
-
- // Set the necessary objects
- QueryInfo queryInfo = new QueryInfo(BaseTests.queryIdentifier, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize,
- queryType, useExpLookupTable, embedSelector, useHDFSExpLookupTable);
-
- if (SystemConfiguration.getBooleanProperty("pir.embedQuerySchema", false))
- {
- queryInfo.addQuerySchema(qSchema);
- }
-
- Paillier paillier = new Paillier(BaseTests.paillierBitSize, BaseTests.certainty);
-
- // Perform the encryption
- logger.info("Performing encryption of the selectors - forming encrypted query vectors:");
- EncryptQuery encryptQuery = new EncryptQuery(queryInfo, selectors, paillier);
- encryptQuery.encrypt(numThreads);
- logger.info("Completed encryption of the selectors - completed formation of the encrypted query vectors:");
-
- // Dork with the embedSelectorMap to generate a false positive for the last valid selector in selectors
- if (testFalsePositive)
- {
- Querier querier = encryptQuery.getQuerier();
- HashMap<Integer,String> embedSelectorMap = querier.getEmbedSelectorMap();
- logger.info("embedSelectorMap((embedSelectorMap.size()-2)) = " + embedSelectorMap.get((embedSelectorMap.size() - 2)) + " selector = "
- + selectors.get((embedSelectorMap.size() - 2)));
- embedSelectorMap.put((embedSelectorMap.size() - 2), "fakeEmbeddedSelector");
- }
-
- // Write necessary output files
- storage.store(fileQuerier, encryptQuery.getQuerier());
- storage.store(fileQuery, encryptQuery.getQuery());
-
- // Perform the PIR query and build the response elements
- logger.info("Performing the PIR Query and constructing the response elements:");
- Query query = storage.recall(fileQuery, Query.class);
- Responder pirResponder = new Responder(query);
- logger.info("Query and Responder elements constructed");
- for (JSONObject jsonData : dataElements)
- {
- String selector = QueryUtils.getSelectorByQueryTypeJSON(qSchema, jsonData);
- logger.info("selector = " + selector + " numDataElements = " + jsonData.size());
- try
- {
- pirResponder.addDataElement(selector, jsonData);
- } catch (Exception e)
- {
- fail(e.toString());
- }
- }
- logger.info("Completed the PIR Query and construction of the response elements:");
-
- // Set the response object, extract, write to file
- logger.info("Forming response from response elements; writing to a file");
- pirResponder.setResponseElements();
- Response responseOut = pirResponder.getResponse();
- storage.store(fileResponse, responseOut);
- logger.info("Completed forming response from response elements and writing to a file");
-
- // Perform decryption
- // Reconstruct the necessary objects from the files
- logger.info("Performing decryption; writing final results file");
- Response responseIn = storage.recall(fileResponse, Response.class);
- Querier querier = storage.recall(fileQuerier, Querier.class);
-
- // Perform decryption and output the result file
- DecryptResponse decryptResponse = new DecryptResponse(responseIn, querier);
- decryptResponse.decrypt(numThreads);
- decryptResponse.writeResultFile(fileFinalResults);
- logger.info("Completed performing decryption and writing final results file");
-
- // Read in results
- logger.info("Reading in and checking results");
- List<QueryResponseJSON> results = TestUtils.readResultsFile(fileFinalResults);
-
- // Clean up
- fileQuerier.delete();
- fileQuery.delete();
- fileResponse.delete();
- fileFinalResults.delete();
-
- return results;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/main/java/org/apache/pirk/test/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/test/utils/TestUtils.java b/src/main/java/org/apache/pirk/test/utils/TestUtils.java
deleted file mode 100644
index 1ea01fb..0000000
--- a/src/main/java/org/apache/pirk/test/utils/TestUtils.java
+++ /dev/null
@@ -1,312 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.test.utils;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.pirk.schema.response.QueryResponseJSON;
-import org.apache.pirk.utils.SystemConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-
-/**
- * Class to hold testing utilities
- *
- */
-public class TestUtils
-{
- private static final Logger logger = LoggerFactory.getLogger(TestUtils.class);
-
- /**
- * Method to delete an ES index
- */
- public static void deleteESTestIndex(String index)
- {
- logger.info("Deleting index:");
- ProcessBuilder pDelete = new ProcessBuilder("curl", "-XDELETE", index);
- try
- {
- executeCommand(pDelete);
- } catch (IOException e)
- {
- e.printStackTrace();
- }
- }
-
- /**
- * Method to execute process
- */
- public static void executeCommand(ProcessBuilder p) throws IOException
- {
- Process proc = p.start();
-
- try (BufferedReader stdInput = new BufferedReader(new InputStreamReader(proc.getInputStream()));
- BufferedReader stdError = new BufferedReader(new InputStreamReader(proc.getErrorStream())))
- {
- // Read the output from the command
- logger.info("Standard output of the command:\n");
- String s;
- while ((s = stdInput.readLine()) != null)
- {
- logger.info(s);
- }
-
- // Read any errors from the attempted command
- logger.info("Standard error of the command (if any):\n");
- while ((s = stdError.readLine()) != null)
- {
- logger.info(s);
- }
- }
- }
-
- /**
- * Helper method to add elements to the test data schema
- */
- public static void addElement(Document doc, Element rootElement, String elementName, String typeIn, String isArrayIn, String partitionerIn)
- {
- Element element = doc.createElement("element");
- rootElement.appendChild(element);
-
- Element name = doc.createElement("name");
- name.appendChild(doc.createTextNode(elementName));
- element.appendChild(name);
-
- Element type = doc.createElement("type");
- type.appendChild(doc.createTextNode(typeIn));
- element.appendChild(type);
-
- if (isArrayIn.equals("true"))
- {
- element.appendChild(doc.createElement("isArray"));
- }
-
- if (partitionerIn != null)
- {
- Element partitioner = doc.createElement("partitioner");
- partitioner.appendChild(doc.createTextNode(partitionerIn));
- element.appendChild(partitioner);
- }
- }
-
- /**
- * Creates the test query schema file
- */
- public static void createQuerySchema(String schemaFile, String querySchemaName, String dataSchemaNameInput, String selectorNameInput,
- List<String> elementNames, List<String> filterNames, String filter) throws IOException
- {
- createQuerySchema(schemaFile, querySchemaName, dataSchemaNameInput, selectorNameInput, elementNames, filterNames, filter, true, null, false);
- }
-
- /**
- * Creates the test query schema file
- */
- public static void createQuerySchema(String schemaFile, String querySchemaName, String dataSchemaNameInput, String selectorNameInput,
- List<String> elementNames, List<String> filterNames, String filter, boolean append, FileSystem fs, boolean hdfs) throws IOException
- {
- logger.info("createQuerySchema: querySchemaName = " + querySchemaName);
-
- // Create a temporary file for the test schema, set in the properties
- String fileName;
- File file = null;
- OutputStreamWriter osw = null;
- if (hdfs)
- {
- Path filePath = new Path(schemaFile);
- fs.deleteOnExit(filePath);
- fileName = filePath.toString();
-
- osw = new OutputStreamWriter(fs.create(filePath, true));
-
- logger.info("hdfs: filePath = " + fileName);
- }
- else
- {
- file = File.createTempFile(schemaFile, ".xml");
- file.deleteOnExit();
- fileName = file.toString();
- logger.info("localFS: file = " + file.toString());
- }
-
- if (append)
- {
- String currentSchemas = SystemConfiguration.getProperty("query.schemas", "");
- if (currentSchemas.equals("") || currentSchemas.equals("none"))
- {
- SystemConfiguration.setProperty("query.schemas", fileName);
- }
- else
- {
- SystemConfiguration.setProperty("query.schemas", SystemConfiguration.getProperty("query.schemas", "") + "," + fileName);
- }
- }
- logger.info("query.schemas = " + SystemConfiguration.getProperty("query.schemas"));
-
- // Write to the file
- try
- {
- DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
- DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
- Document doc = dBuilder.newDocument();
-
- // root element
- Element rootElement = doc.createElement("schema");
- doc.appendChild(rootElement);
-
- // Add the schemaName
- Element schemaNameElement = doc.createElement("schemaName");
- schemaNameElement.appendChild(doc.createTextNode(querySchemaName));
- rootElement.appendChild(schemaNameElement);
-
- // Add the dataSchemaName
- Element dataSchemaNameElement = doc.createElement("dataSchemaName");
- dataSchemaNameElement.appendChild(doc.createTextNode(dataSchemaNameInput));
- rootElement.appendChild(dataSchemaNameElement);
-
- // Add the selectorName
- Element selectorNameElement = doc.createElement("selectorName");
- selectorNameElement.appendChild(doc.createTextNode(selectorNameInput));
- rootElement.appendChild(selectorNameElement);
-
- // Add the elementNames
- Element elements = doc.createElement("elements");
- rootElement.appendChild(elements);
- for (String elementName : elementNames)
- {
- logger.info("elementName = " + elementName);
- Element name = doc.createElement("name");
- name.appendChild(doc.createTextNode(elementName));
- elements.appendChild(name);
- }
-
- // Add the filter
- if (filter != null)
- {
- Element filterElement = doc.createElement("filter");
- filterElement.appendChild(doc.createTextNode(filter));
- rootElement.appendChild(filterElement);
-
- // Add the filterNames
- Element filterNamesElement = doc.createElement("filterNames");
- rootElement.appendChild(filterNamesElement);
- for (String filterName : filterNames)
- {
- logger.info("filterName = " + filterName);
- Element name = doc.createElement("name");
- name.appendChild(doc.createTextNode(filterName));
- filterNamesElement.appendChild(name);
- }
- }
-
- // Write to a xml file
- TransformerFactory transformerFactory = TransformerFactory.newInstance();
- Transformer transformer = transformerFactory.newTransformer();
- DOMSource source = new DOMSource(doc);
- StreamResult result;
- if (hdfs)
- {
- result = new StreamResult(osw);
- }
- else
- {
- result = new StreamResult(file);
- }
- transformer.transform(source, result);
-
- // Output for testing
- StreamResult consoleResult = new StreamResult(System.out);
- transformer.transform(source, consoleResult);
- System.out.println();
-
- if (osw != null)
- {
- osw.close();
- }
-
- } catch (Exception e)
- {
- e.printStackTrace();
- }
- }
-
- /**
- * Converts the result file into an ArrayList of QueryResponseJSON objects
- */
- public static List<QueryResponseJSON> readResultsFile(File file)
- {
- List<QueryResponseJSON> results = new ArrayList<>();
- try (BufferedReader br = new BufferedReader(new FileReader(file)))
- {
- String line;
- while ((line = br.readLine()) != null)
- {
- QueryResponseJSON jsonResult = new QueryResponseJSON(line);
- results.add(jsonResult);
- }
- } catch (Exception e)
- {
- logger.error(e.toString());
- }
-
- return results;
- }
-
- /**
- * Write the ArrayList<String to a tmp file in the local filesystem with the given fileName
- *
- */
- public static String writeToTmpFile(List<String> list, String fileName, String suffix) throws IOException
- {
- File file = File.createTempFile(fileName, suffix);
- file.deleteOnExit();
- logger.info("localFS: file = " + file);
-
- FileWriter fw = new FileWriter(file);
- try (BufferedWriter bw = new BufferedWriter(fw))
- {
- for (String s : list)
- {
- bw.write(s);
- bw.newLine();
- }
- }
-
- return file.getPath();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/general/ISO8601DateParserTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/general/ISO8601DateParserTest.java b/src/test/java/org/apache/pirk/general/ISO8601DateParserTest.java
deleted file mode 100644
index 786f289..0000000
--- a/src/test/java/org/apache/pirk/general/ISO8601DateParserTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.general;
-
-import static org.junit.Assert.assertEquals;
-
-import java.text.ParseException;
-
-import org.apache.pirk.utils.ISO8601DateParser;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Class to test basic functionality of ISO8601DateParser class
- */
-public class ISO8601DateParserTest
-{
- private static final Logger logger = LoggerFactory.getLogger(ISO8601DateParserTest.class);
-
- @Test
- public void testDateParsing() throws ParseException
- {
- logger.info("Starting testDateParsing: ");
-
- String date = "2016-02-20T23:29:05.000Z";
- long longDate = Long.parseLong("1456010945000"); // date in UTC
-
- assertEquals(longDate, ISO8601DateParser.getLongDate(date));
- assertEquals(date, ISO8601DateParser.fromLongDate(longDate));
-
- logger.info("Successfully completed testDateParsing");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/general/KeyedHashTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/general/KeyedHashTest.java b/src/test/java/org/apache/pirk/general/KeyedHashTest.java
deleted file mode 100644
index 676609f..0000000
--- a/src/test/java/org/apache/pirk/general/KeyedHashTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.general;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.pirk.utils.KeyedHash;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Basic functional tests for KeyedHash
- *
- */
-public class KeyedHashTest
-{
- private static final Logger logger = LoggerFactory.getLogger(KeyedHashTest.class);
-
- @Test
- public void testKeyedHash()
- {
- logger.info("Starting testKeyedHash: ");
-
- int hash1 = KeyedHash.hash("someKey", 12, "someInput");
- logger.info("hash1 = " + hash1 + " hash1 = " + Integer.toString(hash1, 2));
-
- int hash2 = KeyedHash.hash("someKey", 32, "someInput");
- logger.info("hash2 = " + hash2 + " hash2 = " + Integer.toString(hash2, 2));
-
- int hash3 = KeyedHash.hash("someKey", 34, "someInput");
- logger.info("hash3 = " + hash3 + " hash3 = " + Integer.toString(hash3, 2));
-
- assertEquals(hash2, hash3);
- assertEquals(hash1, hash2 & 0xFFF);
-
- logger.info("Successfully completed testKeyedHash");
- }
-
- @Test
- public void testKeyedHashWithType()
- {
- testKeyedHashType("MD5");
- testKeyedHashType("SHA-1");
- testKeyedHashType("SHA-256");
- testKeyedHashType("FAKE-HASH-TYPE");
- }
-
- private void testKeyedHashType(String type)
- {
- logger.info("Starting testKeyedHashType with type: " + type);
-
- int hash1 = KeyedHash.hash("someKey", 12, "someInput", type);
- logger.info("hash1 = " + hash1 + " hash1 = " + Integer.toString(hash1, 2));
-
- int hash2 = KeyedHash.hash("someKey", 32, "someInput", type);
- logger.info("hash2 = " + hash2 + " hash2 = " + Integer.toString(hash2, 2));
-
- int hash3 = KeyedHash.hash("someKey", 34, "someInput", type);
- logger.info("hash3 = " + hash3 + " hash3 = " + Integer.toString(hash3, 2));
-
- assertEquals(hash2, hash3);
- assertEquals(hash1, hash2 & 0xFFF);
-
- logger.info("Successfully completed testKeyedHashType with type: " + type);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/general/PaillierTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/general/PaillierTest.java b/src/test/java/org/apache/pirk/general/PaillierTest.java
deleted file mode 100644
index 14347fa..0000000
--- a/src/test/java/org/apache/pirk/general/PaillierTest.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.general;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
-
-import java.math.BigInteger;
-import java.util.Random;
-
-import org.apache.pirk.encryption.Paillier;
-import org.apache.pirk.utils.PIRException;
-import org.apache.pirk.utils.SystemConfiguration;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Basic test functionality for Paillier library
- *
- */
-public class PaillierTest
-{
- private static final Logger logger = LoggerFactory.getLogger(PaillierTest.class);
-
- private static BigInteger p = null; // large prime
- private static BigInteger q = null; // large prime
- private static BigInteger N = null; // N=pq, RSA modulus
- private static BigInteger NSquared = null; // N^2
- private static BigInteger lambdaN = null; // lambda(N) = lcm(p-1,q-1)
-
- private static int bitLength = 0; // bit length of the modulus N
- private static int certainty = 64; // prob that new BigInteger values represents primes will exceed (1 - (1/2)^certainty)
-
- private static BigInteger r1 = null; // random number in (Z/NZ)*
- private static BigInteger r2 = null; // random number in (Z/NZ)*
-
- private static BigInteger m1 = null; // message to encrypt
- private static BigInteger m2 = null; // message to encrypt
-
- @BeforeClass
- public static void setup()
- {
- p = BigInteger.valueOf(7);
- q = BigInteger.valueOf(17);
- N = p.multiply(q);
- NSquared = N.multiply(N);
-
- lambdaN = BigInteger.valueOf(48);
-
- r1 = BigInteger.valueOf(3);
- r2 = BigInteger.valueOf(4);
-
- m1 = BigInteger.valueOf(5);
- m2 = BigInteger.valueOf(2);
-
- bitLength = 201;// bitLength = 384;
- certainty = 128;
-
- logger.info("p = " + p.intValue() + " q = " + q.intValue() + " N = " + N.intValue() + " bitLength = " + N.bitLength() + " lambdaN = " + lambdaN + " m1 = "
- + m1.intValue() + " m2 = " + m2.intValue() + " r1 = " + r1.intValue() + " r2 = " + r2.intValue());
- }
-
- @Test
- public void testPIRExceptions()
- {
- try
- {
- Paillier paillier = new Paillier(BigInteger.valueOf(2), BigInteger.valueOf(2), 128);
- assertNotNull(paillier);
- fail("Paillier constructor did not throw PIRException for p,q < 3");
- } catch (PIRException ignore)
- {}
-
- try
- {
- Paillier paillier = new Paillier(BigInteger.valueOf(2), BigInteger.valueOf(3), 128);
- assertNotNull(paillier);
- fail("Paillier constructor did not throw PIRException for p < 3");
- } catch (PIRException ignore)
- {}
-
- try
- {
- Paillier paillier = new Paillier(BigInteger.valueOf(3), BigInteger.valueOf(2), 128);
- assertNotNull(paillier);
- fail("Paillier constructor did not throw PIRException for q < 3");
- } catch (PIRException ignore)
- {}
-
- try
- {
- Paillier paillier = new Paillier(BigInteger.valueOf(7), BigInteger.valueOf(7), 128);
- assertNotNull(paillier);
- fail("Paillier constructor did not throw PIRException for p = q");
- } catch (PIRException ignore)
- {}
-
- try
- {
- Paillier paillier = new Paillier(BigInteger.valueOf(8), BigInteger.valueOf(7), 128);
- assertNotNull(paillier);
- fail("Paillier constructor did not throw PIRException for p not prime");
- } catch (PIRException ignore)
- {}
-
- try
- {
- Paillier paillier = new Paillier(BigInteger.valueOf(7), BigInteger.valueOf(10), 128);
- assertNotNull(paillier);
- fail("Paillier constructor did not throw PIRException for q not prime");
- } catch (PIRException ignore)
- {}
-
- try
- {
- int systemPrimeCertainty = SystemConfiguration.getIntProperty("pir.primeCertainty", 128);
- Paillier paillier = new Paillier(3072, systemPrimeCertainty - 10);
- assertNotNull(paillier);
- fail("Paillier constructor did not throw PIRException for certainty less than system default of " + systemPrimeCertainty);
- } catch (PIRException ignore)
- {}
-
- try
- {
- Paillier pailler = new Paillier(p, q, bitLength);
- BigInteger encM1 = pailler.encrypt(N);
- assertNotNull(encM1);
- fail("Paillier encryption did not throw PIRException for message m = N");
- } catch (PIRException ignore)
- {}
-
- try
- {
- Paillier pailler = new Paillier(p, q, bitLength);
- BigInteger encM1 = pailler.encrypt(N.add(BigInteger.TEN));
- assertNotNull(encM1);
- fail("Paillier encryption did not throw PIRException for message m > N");
- } catch (PIRException ignore)
- {}
-
- try
- {
- Paillier pailler = new Paillier(bitLength, 128, bitLength);
- assertNotNull(pailler);
- fail("Paillier constructor did not throw PIRException for ensureBitSet = bitLength");
- } catch (PIRException ignore)
- {}
-
- try
- {
- Paillier pailler = new Paillier(bitLength, 128, bitLength + 1);
- assertNotNull(pailler);
- fail("Paillier constructor did not throw PIRException for ensureBitSet > bitLength");
- } catch (PIRException ignore)
- {}
- }
-
- @Test
- public void testPaillierGivenAllParameters() throws Exception
- {
- logger.info("Starting testPaillierGivenAllParameters: ");
-
- Paillier pailler = new Paillier(p, q, bitLength);
-
- assertEquals(pailler.getN(), N);
- assertEquals(pailler.getLambdaN(), lambdaN);
-
- // Check encryption
- BigInteger encM1 = pailler.encrypt(m1, r1);
- BigInteger encM2 = pailler.encrypt(m2, r2);
- logger.info("encM1 = " + encM1.intValue() + " encM2 = " + encM2.intValue());
-
- assertEquals(encM1, BigInteger.valueOf(14019));
- assertEquals(encM2, BigInteger.valueOf(8836));
-
- // Check decryption
- BigInteger decM1 = pailler.decrypt(encM1);
- BigInteger decM2 = pailler.decrypt(encM2);
- logger.info("decM1 = " + decM1.intValue() + " decM2 = " + decM2.intValue());
-
- assertEquals(decM1, m1);
- assertEquals(decM2, m2);
-
- // Check homomorphic property: E_r1(m1)*E_r2(m2) mod N^2 = E_r1r2((m1+m2) mod N) mod N^2
- BigInteger encM1_times_encM2 = (encM1.multiply(encM2)).mod(NSquared);
- BigInteger encM1plusM2 = pailler.encrypt((m1.add(m2)).mod(N), r1.multiply(r2));
- logger.info("encM1_times_encM2 = " + encM1_times_encM2.intValue() + " encM1plusM2 = " + encM1plusM2.intValue());
-
- assertEquals(encM1_times_encM2, BigInteger.valueOf(5617));
- assertEquals(encM1plusM2, BigInteger.valueOf(5617));
-
- logger.info("Successfully completed testPaillierGivenAllParameters: ");
- }
-
- @Test
- public void testPaillierWithKeyGeneration() throws Exception
- {
- logger.info("Starting testPaillierWithKeyGeneration: ");
-
- // Test with and without gmp optimization for modPow
- SystemConfiguration.setProperty("pallier.FIPSPrimeGenerationChecks", "true");
- SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
- SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "true");
- testPaillerWithKeyGenerationGeneral();
-
- SystemConfiguration.setProperty("pallier.FIPSPrimeGenerationChecks", "false");
-
- SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
- SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "true");
- testPaillerWithKeyGenerationGeneral();
-
- SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
- SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "false");
- testPaillerWithKeyGenerationGeneral();
-
- SystemConfiguration.setProperty("paillier.useGMPForModPow", "false");
- SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "false");
- testPaillerWithKeyGenerationGeneral();
-
- // Reset the properties
- SystemConfiguration.initialize();
-
- logger.info("Ending testPaillierWithKeyGeneration: ");
- }
-
- public void testPaillerWithKeyGenerationGeneral() throws Exception
- {
- // Test without requiring highest bit to be set
- logger.info("Starting testPaillierWithKeyGenerationBitSetOption with ensureHighBitSet = false");
- testPaillierWithKeyGenerationBitSetOption(-1);
-
- // Test requiring highest bit to be set
- logger.info("Starting testPaillierWithKeyGenerationBitSetOption with ensureHighBitSet = true");
- testPaillierWithKeyGenerationBitSetOption(5);
- }
-
- public void testPaillierWithKeyGenerationBitSetOption(int ensureBitSet) throws Exception
- {
- Random r = new Random();
- int lowBitLength = 3073; // inclusive
- int highBitLength = 7001; // exclusive
-
- int loopVal = 1; // int loopVal = 1000; //change this and re-test for high loop testing
- for (int i = 0; i < loopVal; ++i)
- {
- logger.info("i = " + i);
-
- basicTestPaillierWithKeyGeneration(bitLength, certainty, ensureBitSet);
- basicTestPaillierWithKeyGeneration(3072, certainty, ensureBitSet);
-
- // Test with random bit length between 3073 and 7000
- int randomLargeBitLength = r.nextInt(highBitLength - lowBitLength) + lowBitLength;
- basicTestPaillierWithKeyGeneration(randomLargeBitLength, certainty, ensureBitSet);
- }
- }
-
- private void basicTestPaillierWithKeyGeneration(int bitLengthInput, int certaintyInput, int ensureBitSet) throws Exception
- {
- Paillier pailler = new Paillier(bitLengthInput, certaintyInput, ensureBitSet);
- BigInteger generatedN = pailler.getN();
- BigInteger geneartedNsquared = generatedN.multiply(generatedN);
-
- // Check the decrypting the encryption yields the message
- BigInteger encM1 = pailler.encrypt(m1);
- BigInteger encM2 = pailler.encrypt(m2);
- logger.info("encM1 = " + encM1.intValue() + " encM2 = " + encM2.intValue());
-
- BigInteger decM1 = pailler.decrypt(encM1);
- BigInteger decM2 = pailler.decrypt(encM2);
- logger.info("decM1 = " + decM1.intValue() + " decM2 = " + decM2.intValue());
-
- assertEquals(decM1, m1);
- assertEquals(decM2, m2);
-
- // Check homomorphic property: E_r1(m1)*E_r2(m2) mod N^2 = E_r1r2((m1+m2) mod N) mod N^2
- BigInteger encM1_times_encM2 = (encM1.multiply(encM2)).mod(geneartedNsquared);
- BigInteger multDecrypt = pailler.decrypt(encM1_times_encM2);
- BigInteger m1_plus_m2 = (m1.add(m2)).mod(N);
-
- logger.info("encM1_times_encM2 = " + encM1_times_encM2.intValue() + " multDecrypt = " + multDecrypt.intValue() + " m1_plus_m2 = " + m1_plus_m2.intValue());
-
- assertEquals(multDecrypt, m1_plus_m2);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/general/PartitionUtilsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/general/PartitionUtilsTest.java b/src/test/java/org/apache/pirk/general/PartitionUtilsTest.java
deleted file mode 100644
index 6f779cd..0000000
--- a/src/test/java/org/apache/pirk/general/PartitionUtilsTest.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.general;
-
-import org.apache.pirk.schema.data.partitioner.IPDataPartitioner;
-import org.apache.pirk.schema.data.partitioner.ISO8601DatePartitioner;
-import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner;
-import org.apache.pirk.utils.PIRException;
-import org.apache.pirk.utils.SystemConfiguration;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.math.BigInteger;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * Class to functionally test the bit conversion utils
- */
-public class PartitionUtilsTest
-{
- private static final Logger logger = LoggerFactory.getLogger(PartitionUtilsTest.class);
-
- @Test
- public void testMask()
- {
- logger.info("Starting testMask: ");
-
- assertEquals(0, PrimitiveTypePartitioner.formBitMask(0).intValue());
-
- assertEquals(0b000000000000001, PrimitiveTypePartitioner.formBitMask(1).intValue());
- assertEquals(0b000000000001111, PrimitiveTypePartitioner.formBitMask(4).intValue());
- assertEquals(0b000000001111111, PrimitiveTypePartitioner.formBitMask(7).intValue());
- assertEquals(0b111111111111111, PrimitiveTypePartitioner.formBitMask(15).intValue());
-
- assertEquals(new BigInteger("FFFFF", 16), PrimitiveTypePartitioner.formBitMask(20));
- assertEquals(new BigInteger("FFFFFFFF", 16), PrimitiveTypePartitioner.formBitMask(32));
- assertEquals(new BigInteger("3FFFFFFFFFF", 16), PrimitiveTypePartitioner.formBitMask(42));
- assertEquals(new BigInteger("7FFFFFFFFFFFFFFF", 16), PrimitiveTypePartitioner.formBitMask(63));
-
- logger.info("Successfully completed testMask");
- }
-
- @Test
- public void testPartitionBits() throws PIRException
- {
- logger.info("Starting testPartitionBits: ");
-
- BigInteger value = new BigInteger("245"); // 11110101
- BigInteger value2 = new BigInteger("983"); // 1111010111
-
- BigInteger mask4 = PrimitiveTypePartitioner.formBitMask(4); // 1111
- BigInteger mask8 = PrimitiveTypePartitioner.formBitMask(8); // 11111111
-
- List<BigInteger> partitions = PrimitiveTypePartitioner.partitionBits(value, 4, mask4);
- assertEquals(2, partitions.size());
- assertEquals(0b1111, partitions.get(0).intValue());
- assertEquals(0b0101, partitions.get(1).intValue());
-
- partitions = PrimitiveTypePartitioner.partitionBits(value2, 4, mask4);
- assertEquals(3, partitions.size());
- assertEquals(0b1111, partitions.get(0).intValue());
- assertEquals(0b0101, partitions.get(1).intValue());
- assertEquals(0b0011, partitions.get(2).intValue());
-
- partitions = PrimitiveTypePartitioner.partitionBits(value, 8, mask8);
- assertEquals(1, partitions.size());
- assertEquals(0b11110101, partitions.get(0).intValue());
-
- try
- {
- partitions = PrimitiveTypePartitioner.partitionBits(value, 4, mask8);
- fail("BitConversionUtils.partitionBits did not throw error for mismatched partitionSize and mask size");
- } catch (Exception ignore)
- {
- // Expected.
- }
-
- logger.info("Successfully completed testPartitionBits");
- }
-
- @Test
- public void testPartitions() throws Exception
- {
- logger.info("Starting testToPartitions:");
-
- PrimitiveTypePartitioner primitivePartitioner = new PrimitiveTypePartitioner();
- IPDataPartitioner ipPartitioner = new IPDataPartitioner();
- ISO8601DatePartitioner datePartitioner = new ISO8601DatePartitioner();
-
- // Test IP
- String ipTest = "127.0.0.1";
- List<BigInteger> partsIP = ipPartitioner.toPartitions(ipTest, PrimitiveTypePartitioner.STRING);
- assertEquals(4, partsIP.size());
- assertEquals(ipTest, ipPartitioner.fromPartitions(partsIP, 0, PrimitiveTypePartitioner.STRING));
-
- // Test Date
- String dateTest = "2016-02-20T23:29:05.000Z";
- List<BigInteger> partsDate = datePartitioner.toPartitions(dateTest, null);
- assertEquals(8, partsDate.size());
- assertEquals(dateTest, datePartitioner.fromPartitions(partsDate, 0, null));
-
- // Test byte
- byte bTest = Byte.parseByte("10");
- List<BigInteger> partsByte = primitivePartitioner.toPartitions(bTest, PrimitiveTypePartitioner.BYTE);
- assertEquals(1, partsByte.size());
- assertEquals(bTest, primitivePartitioner.fromPartitions(partsByte, 0, PrimitiveTypePartitioner.BYTE));
-
- partsByte = primitivePartitioner.toPartitions("12", PrimitiveTypePartitioner.BYTE);
- assertEquals(1, partsByte.size());
- assertEquals((byte) 12, primitivePartitioner.fromPartitions(partsByte, 0, PrimitiveTypePartitioner.BYTE));
-
- List<BigInteger> partsByteMax = primitivePartitioner.toPartitions(Byte.MAX_VALUE, PrimitiveTypePartitioner.BYTE);
- assertEquals(1, partsByteMax.size());
- assertEquals(Byte.MAX_VALUE, primitivePartitioner.fromPartitions(partsByteMax, 0, PrimitiveTypePartitioner.BYTE));
-
- // Test string
- String stringBits = SystemConfiguration.getProperty("pir.stringBits");
- SystemConfiguration.setProperty("pir.stringBits", "64");
- testString("testString"); // over the allowed bit size
- testString("t"); // under the allowed bit size
- SystemConfiguration.setProperty("pir.stringBits", stringBits);
-
- // Test short
- short shortTest = Short.valueOf("2456");
- List<BigInteger> partsShort = primitivePartitioner.toPartitions(shortTest, PrimitiveTypePartitioner.SHORT);
- assertEquals(2, partsShort.size());
- assertEquals(shortTest, primitivePartitioner.fromPartitions(partsShort, 0, PrimitiveTypePartitioner.SHORT));
-
- partsShort = primitivePartitioner.toPartitions("32767", PrimitiveTypePartitioner.SHORT);
- assertEquals(2, partsShort.size());
- assertEquals((short) 32767, primitivePartitioner.fromPartitions(partsShort, 0, PrimitiveTypePartitioner.SHORT));
-
- partsShort = primitivePartitioner.toPartitions((short) -42, PrimitiveTypePartitioner.SHORT);
- assertEquals(2, partsShort.size());
- assertEquals((short) -42, primitivePartitioner.fromPartitions(partsShort, 0, PrimitiveTypePartitioner.SHORT));
-
- List<BigInteger> partsShortMax = primitivePartitioner.toPartitions(Short.MAX_VALUE, PrimitiveTypePartitioner.SHORT);
- assertEquals(2, partsShortMax.size());
- assertEquals(Short.MAX_VALUE, primitivePartitioner.fromPartitions(partsShortMax, 0, PrimitiveTypePartitioner.SHORT));
-
- // Test int
- int intTest = Integer.parseInt("-5789");
- List<BigInteger> partsInt = primitivePartitioner.toPartitions(intTest, PrimitiveTypePartitioner.INT);
- assertEquals(4, partsInt.size());
- assertEquals(intTest, primitivePartitioner.fromPartitions(partsInt, 0, PrimitiveTypePartitioner.INT));
-
- partsInt = primitivePartitioner.toPartitions("2016", PrimitiveTypePartitioner.INT);
- assertEquals(4, partsInt.size());
- assertEquals(2016, primitivePartitioner.fromPartitions(partsInt, 0, PrimitiveTypePartitioner.INT));
-
- partsInt = primitivePartitioner.toPartitions(1386681237, PrimitiveTypePartitioner.INT);
- assertEquals(4, partsInt.size());
- assertEquals(1386681237, primitivePartitioner.fromPartitions(partsInt, 0, PrimitiveTypePartitioner.INT));
-
- List<BigInteger> partsIntMax = primitivePartitioner.toPartitions(Integer.MAX_VALUE, PrimitiveTypePartitioner.INT);
- assertEquals(4, partsIntMax.size());
- assertEquals(Integer.MAX_VALUE, primitivePartitioner.fromPartitions(partsIntMax, 0, PrimitiveTypePartitioner.INT));
-
- // Test long
- long longTest = Long.parseLong("56789");
- List<BigInteger> partsLong = primitivePartitioner.toPartitions(longTest, PrimitiveTypePartitioner.LONG);
- assertEquals(8, partsLong.size());
- assertEquals(longTest, primitivePartitioner.fromPartitions(partsLong, 0, PrimitiveTypePartitioner.LONG));
-
- List<BigInteger> partsLongMax = primitivePartitioner.toPartitions(Long.MAX_VALUE, PrimitiveTypePartitioner.LONG);
- assertEquals(8, partsLongMax.size());
- assertEquals(Long.MAX_VALUE, primitivePartitioner.fromPartitions(partsLongMax, 0, PrimitiveTypePartitioner.LONG));
-
- // Test float
- float floatTest = Float.parseFloat("567.77");
- List<BigInteger> partsFloat = primitivePartitioner.toPartitions(floatTest, PrimitiveTypePartitioner.FLOAT);
- assertEquals(4, partsFloat.size());
- assertEquals(floatTest, primitivePartitioner.fromPartitions(partsFloat, 0, PrimitiveTypePartitioner.FLOAT));
-
- partsFloat = primitivePartitioner.toPartitions(-99.99f, PrimitiveTypePartitioner.FLOAT);
- assertEquals(4, partsFloat.size());
- assertEquals(-99.99f, primitivePartitioner.fromPartitions(partsFloat, 0, PrimitiveTypePartitioner.FLOAT));
-
- List<BigInteger> partsFloatMax = primitivePartitioner.toPartitions(Float.MAX_VALUE, PrimitiveTypePartitioner.FLOAT);
- assertEquals(4, partsFloatMax.size());
- assertEquals(Float.MAX_VALUE, primitivePartitioner.fromPartitions(partsFloatMax, 0, PrimitiveTypePartitioner.FLOAT));
-
- // Test double
- double doubleTest = Double.parseDouble("567.77");
- List<BigInteger> partsDouble = primitivePartitioner.toPartitions(doubleTest, PrimitiveTypePartitioner.DOUBLE);
- assertEquals(8, partsDouble.size());
- assertEquals(doubleTest, primitivePartitioner.fromPartitions(partsDouble, 0, PrimitiveTypePartitioner.DOUBLE));
-
- List<BigInteger> partsDoubleMax = primitivePartitioner.toPartitions(Double.MAX_VALUE, PrimitiveTypePartitioner.DOUBLE);
- assertEquals(8, partsDoubleMax.size());
- assertEquals(Double.MAX_VALUE, primitivePartitioner.fromPartitions(partsDoubleMax, 0, PrimitiveTypePartitioner.DOUBLE));
-
- // Test char
- char charTest = 'b';
- List<BigInteger> partsChar = primitivePartitioner.toPartitions(charTest, PrimitiveTypePartitioner.CHAR);
- assertEquals(2, partsChar.size());
- assertEquals(charTest, primitivePartitioner.fromPartitions(partsChar, 0, PrimitiveTypePartitioner.CHAR));
-
- // Ensure Endianness preserved
- charTest = '\uFFFE';
- partsChar = primitivePartitioner.toPartitions(charTest, PrimitiveTypePartitioner.CHAR);
- assertEquals(2, partsChar.size());
- assertEquals(charTest, primitivePartitioner.fromPartitions(partsChar, 0, PrimitiveTypePartitioner.CHAR));
-
- charTest = '\uFEFF';
- partsChar = primitivePartitioner.toPartitions(charTest, PrimitiveTypePartitioner.CHAR);
- assertEquals(2, partsChar.size());
- assertEquals(charTest, primitivePartitioner.fromPartitions(partsChar, 0, PrimitiveTypePartitioner.CHAR));
-
- List<BigInteger> partsCharMax = primitivePartitioner.toPartitions(Character.MAX_VALUE, PrimitiveTypePartitioner.CHAR);
- assertEquals(2, partsCharMax.size());
- assertEquals(Character.MAX_VALUE, primitivePartitioner.fromPartitions(partsCharMax, 0, PrimitiveTypePartitioner.CHAR));
-
- logger.info("Sucessfully completed testToPartitions:");
- }
-
- @Test
- public void testPaddedPartitions() throws PIRException
- {
- PrimitiveTypePartitioner primitivePartitioner = new PrimitiveTypePartitioner();
-
- List<String> primitiveTypes = Arrays.asList(PrimitiveTypePartitioner.BYTE, PrimitiveTypePartitioner.CHAR, PrimitiveTypePartitioner.SHORT,
- PrimitiveTypePartitioner.INT, PrimitiveTypePartitioner.LONG, PrimitiveTypePartitioner.FLOAT, PrimitiveTypePartitioner.DOUBLE,
- PrimitiveTypePartitioner.STRING);
- for (String type : primitiveTypes)
- {
- assertEquals(primitivePartitioner.getNumPartitions(type), primitivePartitioner.getPaddedPartitions(type).size());
- }
- }
-
- private void testString(String testString) throws Exception
- {
- PrimitiveTypePartitioner ptp = new PrimitiveTypePartitioner();
-
- List<BigInteger> partsString = ptp.toPartitions(testString, PrimitiveTypePartitioner.STRING);
- int numParts = Integer.parseInt(SystemConfiguration.getProperty("pir.stringBits")) / 8;
- assertEquals(numParts, partsString.size());
-
- logger.info("testString.getBytes().length = " + testString.getBytes().length);
- int offset = numParts;
- if (testString.getBytes().length < numParts)
- {
- offset = testString.getBytes().length;
- }
- String element = new String(testString.getBytes(), 0, offset);
- assertEquals(element, ptp.fromPartitions(partsString, 0, PrimitiveTypePartitioner.STRING));
- }
-}
[3/6] incubator-pirk git commit: Move all tests under src/test
directory, this closes apache/incubator-pirk#66
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java b/src/test/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java
new file mode 100644
index 0000000..58f835c
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java
@@ -0,0 +1,458 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.distributed.testsuite;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.pirk.encryption.Paillier;
+import org.apache.pirk.inputformat.hadoop.InputFormatConst;
+import org.apache.pirk.inputformat.hadoop.json.JSONInputFormatBase;
+import org.apache.pirk.querier.wideskies.Querier;
+import org.apache.pirk.querier.wideskies.decrypt.DecryptResponse;
+import org.apache.pirk.querier.wideskies.encrypt.EncryptQuery;
+import org.apache.pirk.query.wideskies.Query;
+import org.apache.pirk.query.wideskies.QueryInfo;
+import org.apache.pirk.responder.wideskies.ResponderProps;
+import org.apache.pirk.responder.wideskies.mapreduce.ComputeResponseTool;
+import org.apache.pirk.response.wideskies.Response;
+import org.apache.pirk.schema.response.QueryResponseJSON;
+import org.apache.pirk.serialization.HadoopFileSystemStore;
+import org.apache.pirk.test.distributed.DistributedTestDriver;
+import org.apache.pirk.test.utils.BaseTests;
+import org.apache.pirk.test.utils.Inputs;
+import org.apache.pirk.test.utils.TestUtils;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.apache.spark.launcher.SparkLauncher;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Distributed test class for PIR
+ *
+ */
+public class DistTestSuite
+{
+ private static final Logger logger = LoggerFactory.getLogger(DistTestSuite.class);
+
+ // This method also tests all non-query specific configuration options/properties
+ // for the MapReduce version of PIR
+ public static void testJSONInputMR(FileSystem fs, List<JSONObject> dataElements) throws Exception
+ {
+ logger.info("Starting testJSONInputMR");
+
+ // Pull original data and query schema properties
+ SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+ SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
+
+ SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
+ SystemConfiguration.setProperty("pir.maxHitsPerSelector", "100");
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+ // Set up base configs
+ SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.BASE_FORMAT);
+ SystemConfiguration.setProperty("pir.inputData", SystemConfiguration.getProperty(DistributedTestDriver.JSON_PIR_INPUT_FILE_PROPERTY));
+ SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:0");
+
+ // Run tests
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
+ BaseTests.testDNSIPQuery(dataElements, fs, false, true, 1);
+
+ SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 2);
+ BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
+
+ BaseTests.testSRCIPQueryNoFilter(dataElements, fs, false, true, 2);
+
+ // Test hit limits per selector
+ SystemConfiguration.setProperty("pir.limitHitsPerSelector", "true");
+ SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 3);
+ SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
+ SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
+
+ // Test the local cache for modular exponentiation
+ SystemConfiguration.setProperty("pir.useLocalCache", "true");
+ BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
+ BaseTests.testSRCIPQuery(dataElements, fs, false, true, 2);
+ SystemConfiguration.setProperty("pir.useLocalCache", "false");
+
+ // Change query for NXDOMAIN
+ SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:3");
+ BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 2);
+ SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+ BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 2);
+ SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:0");
+
+ // Test the expTable cases
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+
+ // In memory table
+ SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+ SystemConfiguration.setProperty("pirTest.useExpLookupTable", "true");
+ BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
+
+ // Create exp table in hdfs
+ SystemConfiguration.setProperty("mapreduce.map.memory.mb", "10000");
+ SystemConfiguration.setProperty("mapreduce.reduce.memory.mb", "10000");
+ SystemConfiguration.setProperty("mapreduce.map.java.opts", "-Xmx9000m");
+ SystemConfiguration.setProperty("mapreduce.reduce.java.opts", "-Xmx9000m");
+
+ SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "true");
+ SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
+ SystemConfiguration.setProperty("pir.expCreationSplits", "50");
+ SystemConfiguration.setProperty("pir.numExpLookupPartitions", "150");
+ BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
+
+ // Reset exp properties
+ SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+ SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
+
+ // Reset property
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+
+ // Test embedded QuerySchema
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
+
+ logger.info("Completed testJSONInputMR");
+ }
+
+ public static void testESInputMR(FileSystem fs, List<JSONObject> dataElements) throws Exception
+ {
+ logger.info("Starting testESInputMR");
+
+ SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+ SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
+
+ SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
+ SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+ // Set up ES configs
+ SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.ES);
+ SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:0");
+ SystemConfiguration.setProperty("pir.esResource", SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_RESOURCE_PROPERTY));
+
+ // Run tests
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
+ BaseTests.testSRCIPQuery(dataElements, fs, false, true, 2);
+ BaseTests.testDNSIPQuery(dataElements, fs, false, true, 1);
+
+ SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 2);
+ BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
+
+ // Change query for NXDOMAIN
+ SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:3");
+
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+ BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 3);
+ SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+ BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 3);
+
+ logger.info("Completed testESInputMR");
+ }
+
+ public static void testJSONInputSpark(FileSystem fs, List<JSONObject> dataElements) throws Exception
+ {
+ logger.info("Starting testJSONInputSpark");
+
+ SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+ SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
+ SystemConfiguration.setProperty("pir.useModExpJoin", "false");
+
+ SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
+ SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
+
+ SystemConfiguration.setProperty("pir.numColMultPartitions", "20");
+ SystemConfiguration.setProperty("pir.colMultReduceByKey", "false");
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+ // Set up JSON configs
+ SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.BASE_FORMAT);
+ SystemConfiguration.setProperty("pir.inputData", SystemConfiguration.getProperty(DistributedTestDriver.JSON_PIR_INPUT_FILE_PROPERTY));
+ SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:0");
+
+ // Run tests
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
+ BaseTests.testDNSIPQuery(dataElements, fs, true, true, 1);
+
+ SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 2);
+ BaseTests.testDNSIPQuery(dataElements, fs, true, true, 2);
+ BaseTests.testSRCIPQuery(dataElements, fs, true, true, 2);
+
+ BaseTests.testSRCIPQueryNoFilter(dataElements, fs, true, true, 2);
+
+ // Test embedded QuerySchema
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+ // Test pad columns
+ SystemConfiguration.setProperty("pir.padEmptyColumns", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
+ SystemConfiguration.setProperty("pir.padEmptyColumns", "false");
+
+ // Test hit limits per selector
+ SystemConfiguration.setProperty("pir.limitHitsPerSelector", "true");
+ SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 3);
+ SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
+ SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
+
+ // Test the local cache for modular exponentiation
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+ SystemConfiguration.setProperty("pir.useLocalCache", "true");
+ BaseTests.testDNSIPQuery(dataElements, fs, true, true, 3);
+
+ // Test the join functionality for the modular exponentiation table
+ SystemConfiguration.setProperty("pir.useModExpJoin", "true");
+ BaseTests.testDNSIPQuery(dataElements, fs, true, true, 3);
+ SystemConfiguration.setProperty("pir.useModExpJoin", "false");
+
+ // Test file based exp lookup table for modular exponentiation
+ SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "true");
+ SystemConfiguration.setProperty("pir.expCreationSplits", "500");
+ SystemConfiguration.setProperty("pir.numExpLookupPartitions", "150");
+ BaseTests.testDNSIPQuery(dataElements, fs, true, true, 2);
+ SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+
+ // Change query for NXDOMAIN
+ SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:3");
+
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+ BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
+ SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+ BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
+
+ // Test with reduceByKey for column mult
+ SystemConfiguration.setProperty("pir.colMultReduceByKey", "true");
+ BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
+
+ logger.info("Completed testJSONInputSpark");
+ }
+
+ public static void testESInputSpark(FileSystem fs, List<JSONObject> dataElements) throws Exception
+ {
+ logger.info("Starting testESInputSpark");
+
+ SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
+ SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
+
+ SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
+ SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+ // Set up ES configs
+ SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.ES);
+ SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:0");
+ SystemConfiguration.setProperty("pir.esResource", SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_RESOURCE_PROPERTY));
+
+ // Run tests
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
+ BaseTests.testDNSIPQuery(dataElements, fs, true, true, 1);
+ BaseTests.testSRCIPQuery(dataElements, fs, true, true, 2);
+
+ SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+ BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 2);
+ BaseTests.testDNSIPQuery(dataElements, fs, true, true, 2);
+
+ // Change query for NXDOMAIN
+ SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:3");
+
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+ BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
+ SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+ BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
+
+ logger.info("Completed testESInputSpark");
+ }
+
+ // Base method to perform query
+ public static List<QueryResponseJSON> performQuery(String queryType, ArrayList<String> selectors, FileSystem fs, boolean isSpark, int numThreads)
+ throws Exception
+ {
+ logger.info("performQuery: ");
+
+ String queryInputDir = SystemConfiguration.getProperty(DistributedTestDriver.PIR_QUERY_INPUT_DIR);
+ String outputFile = SystemConfiguration.getProperty(DistributedTestDriver.OUTPUT_DIRECTORY_PROPERTY);
+ fs.delete(new Path(outputFile), true); // Ensure old output does not exist.
+
+ SystemConfiguration.setProperty("pir.queryInput", queryInputDir);
+ SystemConfiguration.setProperty("pir.outputFile", outputFile);
+ SystemConfiguration.setProperty("pir.numReduceTasks", "1");
+ SystemConfiguration.setProperty("pir.stopListFile", SystemConfiguration.getProperty(DistributedTestDriver.PIR_STOPLIST_FILE));
+
+ // Create the temp result file
+ File fileFinalResults = File.createTempFile("finalResultsFile", ".txt");
+ fileFinalResults.deleteOnExit();
+ logger.info("fileFinalResults = " + fileFinalResults.getAbsolutePath());
+
+ boolean embedSelector = SystemConfiguration.getBooleanProperty("pirTest.embedSelector", false);
+ boolean useExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useExpLookupTable", false);
+ boolean useHDFSExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useHDFSExpLookupTable", false);
+
+ // Set the necessary objects
+ QueryInfo queryInfo = new QueryInfo(BaseTests.queryIdentifier, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize,
+ queryType, useExpLookupTable, embedSelector, useHDFSExpLookupTable);
+
+ Paillier paillier = new Paillier(BaseTests.paillierBitSize, BaseTests.certainty);
+
+ // Perform the encryption
+ logger.info("Performing encryption of the selectors - forming encrypted query vectors:");
+ EncryptQuery encryptQuery = new EncryptQuery(queryInfo, selectors, paillier);
+ encryptQuery.encrypt(numThreads);
+ logger.info("Completed encryption of the selectors - completed formation of the encrypted query vectors:");
+
+ // Grab the necessary objects
+ Querier querier = encryptQuery.getQuerier();
+ Query query = encryptQuery.getQuery();
+
+ // Write the Querier object to a file
+ Path queryInputDirPath = new Path(queryInputDir);
+ new HadoopFileSystemStore(fs).store(queryInputDirPath, query);
+ fs.deleteOnExit(queryInputDirPath);
+
+ // Grab the original data and query schema properties to reset upon completion
+ String dataSchemaProp = SystemConfiguration.getProperty("data.schemas");
+ String querySchemaProp = SystemConfiguration.getProperty("query.schemas");
+
+ // Get the correct input format class name
+ JSONInputFormatBase jFormat = new JSONInputFormatBase();
+ String jsonBaseInputFormatString = jFormat.getClass().getName();
+ SystemConfiguration.setProperty("pir.baseInputFormat", jsonBaseInputFormatString);
+
+ // Submitting the tool for encrypted query
+ logger.info("Performing encrypted query:");
+ if (isSpark)
+ {
+ // Build args
+ String inputFormat = SystemConfiguration.getProperty("pir.dataInputFormat");
+ logger.info("inputFormat = " + inputFormat);
+ ArrayList<String> args = new ArrayList<>();
+ args.add("-" + ResponderProps.PLATFORM + "=spark");
+ args.add("-" + ResponderProps.DATAINPUTFORMAT + "=" + inputFormat);
+ args.add("-" + ResponderProps.QUERYINPUT + "=" + SystemConfiguration.getProperty("pir.queryInput"));
+ args.add("-" + ResponderProps.OUTPUTFILE + "=" + SystemConfiguration.getProperty("pir.outputFile"));
+ args.add("-" + ResponderProps.STOPLISTFILE + "=" + SystemConfiguration.getProperty("pir.stopListFile"));
+ args.add("-" + ResponderProps.USELOCALCACHE + "=" + SystemConfiguration.getProperty("pir.useLocalCache", "true"));
+ args.add("-" + ResponderProps.LIMITHITSPERSELECTOR + "=" + SystemConfiguration.getProperty("pir.limitHitsPerSelector", "false"));
+ args.add("-" + ResponderProps.MAXHITSPERSELECTOR + "=" + SystemConfiguration.getProperty("pir.maxHitsPerSelector", "1000"));
+ args.add("-" + ResponderProps.QUERYSCHEMAS + "=" + Inputs.HDFS_QUERY_FILES);
+ args.add("-" + ResponderProps.DATASCHEMAS + "=" + Inputs.DATA_SCHEMA_FILE_HDFS);
+ args.add("-" + ResponderProps.NUMEXPLOOKUPPARTS + "=" + SystemConfiguration.getProperty("pir.numExpLookupPartitions", "100"));
+ args.add("-" + ResponderProps.USEMODEXPJOIN + "=" + SystemConfiguration.getProperty("pir.useModExpJoin", "false"));
+ args.add("-" + ResponderProps.NUMCOLMULTPARTITIONS + "=" + SystemConfiguration.getProperty("pir.numColMultPartitions", "20"));
+ args.add("-" + ResponderProps.COLMULTREDUCEBYKEY + "=" + SystemConfiguration.getProperty("pir.colMultReduceByKey", "false"));
+ if (inputFormat.equals(InputFormatConst.BASE_FORMAT))
+ {
+ args.add("-" + ResponderProps.INPUTDATA + "=" + SystemConfiguration.getProperty("pir.inputData"));
+ args.add("-" + ResponderProps.BASEQUERY + "=" + SystemConfiguration.getProperty("pir.baseQuery"));
+ args.add("-" + ResponderProps.BASEINPUTFORMAT + "=" + SystemConfiguration.getProperty("pir.baseInputFormat"));
+ }
+ else if (inputFormat.equals(InputFormatConst.ES))
+ {
+ args.add("-" + ResponderProps.ESQUERY + "=" + SystemConfiguration.getProperty("pir.esQuery"));
+ args.add("-" + ResponderProps.ESRESOURCE + "=" + SystemConfiguration.getProperty("pir.esResource"));
+ }
+
+ for (String arg : args)
+ {
+ logger.info("arg = " + arg);
+ }
+
+ // Run spark application
+ Process sLauncher = new SparkLauncher().setAppResource(SystemConfiguration.getProperty("jarFile"))
+ .setSparkHome(SystemConfiguration.getProperty("spark.home")).setMainClass("org.apache.pirk.responder.wideskies.ResponderDriver")
+ .addAppArgs(args.toArray(new String[args.size()])).setMaster("yarn-cluster").setConf(SparkLauncher.EXECUTOR_MEMORY, "2g")
+ .setConf(SparkLauncher.DRIVER_MEMORY, "2g").setConf(SparkLauncher.EXECUTOR_CORES, "1").launch();
+ sLauncher.waitFor();
+ }
+ else
+ {
+ SystemConfiguration.setProperty("data.schemas", Inputs.DATA_SCHEMA_FILE_HDFS);
+ SystemConfiguration.setProperty("query.schemas", Inputs.HDFS_QUERY_FILES);
+
+ ComputeResponseTool responseTool = new ComputeResponseTool();
+ ToolRunner.run(responseTool, new String[] {});
+ }
+ logger.info("Completed encrypted query");
+
+ // Perform decryption
+ // Reconstruct the necessary objects from the files
+ logger.info("Performing decryption; writing final results file");
+ Response response = new HadoopFileSystemStore(fs).recall(outputFile, Response.class);
+
+ // Perform decryption and output the result file
+ DecryptResponse decryptResponse = new DecryptResponse(response, querier);
+ decryptResponse.decrypt(numThreads);
+ decryptResponse.writeResultFile(fileFinalResults);
+ logger.info("Completed performing decryption and writing final results file");
+
+ // Read in results
+ logger.info("Reading in and checking results");
+ List<QueryResponseJSON> results = TestUtils.readResultsFile(fileFinalResults);
+
+ // Reset data and query schema properties
+ SystemConfiguration.setProperty("data.schemas", dataSchemaProp);
+ SystemConfiguration.setProperty("query.schemas", querySchemaProp);
+
+ // Clean up output dir in hdfs
+ fs.delete(new Path(outputFile), true);
+
+ return results;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/general/ISO8601DateParserTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/general/ISO8601DateParserTest.java b/src/test/java/org/apache/pirk/test/general/ISO8601DateParserTest.java
new file mode 100644
index 0000000..9821db3
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/general/ISO8601DateParserTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.general;
+
+import static org.junit.Assert.assertEquals;
+
+import java.text.ParseException;
+
+import org.apache.pirk.utils.ISO8601DateParser;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class to test basic functionality of ISO8601DateParser class
+ */
+public class ISO8601DateParserTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(ISO8601DateParserTest.class);
+
+ @Test
+ public void testDateParsing() throws ParseException
+ {
+ logger.info("Starting testDateParsing: ");
+
+ String date = "2016-02-20T23:29:05.000Z";
+ long longDate = Long.parseLong("1456010945000"); // date in UTC
+
+ assertEquals(longDate, ISO8601DateParser.getLongDate(date));
+ assertEquals(date, ISO8601DateParser.fromLongDate(longDate));
+
+ logger.info("Successfully completed testDateParsing");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/general/KeyedHashTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/general/KeyedHashTest.java b/src/test/java/org/apache/pirk/test/general/KeyedHashTest.java
new file mode 100644
index 0000000..e53cebf
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/general/KeyedHashTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.general;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.pirk.utils.KeyedHash;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Basic functional tests for KeyedHash
+ *
+ */
+public class KeyedHashTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(KeyedHashTest.class);
+
+ @Test
+ public void testKeyedHash()
+ {
+ logger.info("Starting testKeyedHash: ");
+
+ int hash1 = KeyedHash.hash("someKey", 12, "someInput");
+ logger.info("hash1 = " + hash1 + " hash1 = " + Integer.toString(hash1, 2));
+
+ int hash2 = KeyedHash.hash("someKey", 32, "someInput");
+ logger.info("hash2 = " + hash2 + " hash2 = " + Integer.toString(hash2, 2));
+
+ int hash3 = KeyedHash.hash("someKey", 34, "someInput");
+ logger.info("hash3 = " + hash3 + " hash3 = " + Integer.toString(hash3, 2));
+
+ assertEquals(hash2, hash3);
+ assertEquals(hash1, hash2 & 0xFFF);
+
+ logger.info("Successfully completed testKeyedHash");
+ }
+
+ @Test
+ public void testKeyedHashWithType()
+ {
+ testKeyedHashType("MD5");
+ testKeyedHashType("SHA-1");
+ testKeyedHashType("SHA-256");
+ testKeyedHashType("FAKE-HASH-TYPE");
+ }
+
+ private void testKeyedHashType(String type)
+ {
+ logger.info("Starting testKeyedHashType with type: " + type);
+
+ int hash1 = KeyedHash.hash("someKey", 12, "someInput", type);
+ logger.info("hash1 = " + hash1 + " hash1 = " + Integer.toString(hash1, 2));
+
+ int hash2 = KeyedHash.hash("someKey", 32, "someInput", type);
+ logger.info("hash2 = " + hash2 + " hash2 = " + Integer.toString(hash2, 2));
+
+ int hash3 = KeyedHash.hash("someKey", 34, "someInput", type);
+ logger.info("hash3 = " + hash3 + " hash3 = " + Integer.toString(hash3, 2));
+
+ assertEquals(hash2, hash3);
+ assertEquals(hash1, hash2 & 0xFFF);
+
+ logger.info("Successfully completed testKeyedHashType with type: " + type);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/general/PaillierTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/general/PaillierTest.java b/src/test/java/org/apache/pirk/test/general/PaillierTest.java
new file mode 100644
index 0000000..6aa1396
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/general/PaillierTest.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.general;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import java.math.BigInteger;
+import java.util.Random;
+
+import org.apache.pirk.encryption.Paillier;
+import org.apache.pirk.utils.PIRException;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Basic test functionality for Paillier library
+ *
+ */
+public class PaillierTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(PaillierTest.class);
+
+ private static BigInteger p = null; // large prime
+ private static BigInteger q = null; // large prime
+ private static BigInteger N = null; // N=pq, RSA modulus
+ private static BigInteger NSquared = null; // N^2
+ private static BigInteger lambdaN = null; // lambda(N) = lcm(p-1,q-1)
+
+ private static int bitLength = 0; // bit length of the modulus N
+ private static int certainty = 64; // prob that new BigInteger values represents primes will exceed (1 - (1/2)^certainty)
+
+ private static BigInteger r1 = null; // random number in (Z/NZ)*
+ private static BigInteger r2 = null; // random number in (Z/NZ)*
+
+ private static BigInteger m1 = null; // message to encrypt
+ private static BigInteger m2 = null; // message to encrypt
+
+ @BeforeClass
+ public static void setup()
+ {
+ p = BigInteger.valueOf(7);
+ q = BigInteger.valueOf(17);
+ N = p.multiply(q);
+ NSquared = N.multiply(N);
+
+ lambdaN = BigInteger.valueOf(48);
+
+ r1 = BigInteger.valueOf(3);
+ r2 = BigInteger.valueOf(4);
+
+ m1 = BigInteger.valueOf(5);
+ m2 = BigInteger.valueOf(2);
+
+ bitLength = 201;// bitLength = 384;
+ certainty = 128;
+
+ logger.info("p = " + p.intValue() + " q = " + q.intValue() + " N = " + N.intValue() + " bitLength = " + N.bitLength() + " lambdaN = " + lambdaN + " m1 = "
+ + m1.intValue() + " m2 = " + m2.intValue() + " r1 = " + r1.intValue() + " r2 = " + r2.intValue());
+ }
+
+ @Test
+ public void testPIRExceptions()
+ {
+ try
+ {
+ Paillier paillier = new Paillier(BigInteger.valueOf(2), BigInteger.valueOf(2), 128);
+ assertNotNull(paillier);
+ fail("Paillier constructor did not throw PIRException for p,q < 3");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier paillier = new Paillier(BigInteger.valueOf(2), BigInteger.valueOf(3), 128);
+ assertNotNull(paillier);
+ fail("Paillier constructor did not throw PIRException for p < 3");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier paillier = new Paillier(BigInteger.valueOf(3), BigInteger.valueOf(2), 128);
+ assertNotNull(paillier);
+ fail("Paillier constructor did not throw PIRException for q < 3");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier paillier = new Paillier(BigInteger.valueOf(7), BigInteger.valueOf(7), 128);
+ assertNotNull(paillier);
+ fail("Paillier constructor did not throw PIRException for p = q");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier paillier = new Paillier(BigInteger.valueOf(8), BigInteger.valueOf(7), 128);
+ assertNotNull(paillier);
+ fail("Paillier constructor did not throw PIRException for p not prime");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier paillier = new Paillier(BigInteger.valueOf(7), BigInteger.valueOf(10), 128);
+ assertNotNull(paillier);
+ fail("Paillier constructor did not throw PIRException for q not prime");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ int systemPrimeCertainty = SystemConfiguration.getIntProperty("pir.primeCertainty", 128);
+ Paillier paillier = new Paillier(3072, systemPrimeCertainty - 10);
+ assertNotNull(paillier);
+ fail("Paillier constructor did not throw PIRException for certainty less than system default of " + systemPrimeCertainty);
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier pailler = new Paillier(p, q, bitLength);
+ BigInteger encM1 = pailler.encrypt(N);
+ assertNotNull(encM1);
+ fail("Paillier encryption did not throw PIRException for message m = N");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier pailler = new Paillier(p, q, bitLength);
+ BigInteger encM1 = pailler.encrypt(N.add(BigInteger.TEN));
+ assertNotNull(encM1);
+ fail("Paillier encryption did not throw PIRException for message m > N");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier pailler = new Paillier(bitLength, 128, bitLength);
+ assertNotNull(pailler);
+ fail("Paillier constructor did not throw PIRException for ensureBitSet = bitLength");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier pailler = new Paillier(bitLength, 128, bitLength + 1);
+ assertNotNull(pailler);
+ fail("Paillier constructor did not throw PIRException for ensureBitSet > bitLength");
+ } catch (PIRException ignore)
+ {}
+ }
+
+ @Test
+ public void testPaillierGivenAllParameters() throws Exception
+ {
+ logger.info("Starting testPaillierGivenAllParameters: ");
+
+ Paillier pailler = new Paillier(p, q, bitLength);
+
+ assertEquals(pailler.getN(), N);
+ assertEquals(pailler.getLambdaN(), lambdaN);
+
+ // Check encryption
+ BigInteger encM1 = pailler.encrypt(m1, r1);
+ BigInteger encM2 = pailler.encrypt(m2, r2);
+ logger.info("encM1 = " + encM1.intValue() + " encM2 = " + encM2.intValue());
+
+ assertEquals(encM1, BigInteger.valueOf(14019));
+ assertEquals(encM2, BigInteger.valueOf(8836));
+
+ // Check decryption
+ BigInteger decM1 = pailler.decrypt(encM1);
+ BigInteger decM2 = pailler.decrypt(encM2);
+ logger.info("decM1 = " + decM1.intValue() + " decM2 = " + decM2.intValue());
+
+ assertEquals(decM1, m1);
+ assertEquals(decM2, m2);
+
+ // Check homomorphic property: E_r1(m1)*E_r2(m2) mod N^2 = E_r1r2((m1+m2) mod N) mod N^2
+ BigInteger encM1_times_encM2 = (encM1.multiply(encM2)).mod(NSquared);
+ BigInteger encM1plusM2 = pailler.encrypt((m1.add(m2)).mod(N), r1.multiply(r2));
+ logger.info("encM1_times_encM2 = " + encM1_times_encM2.intValue() + " encM1plusM2 = " + encM1plusM2.intValue());
+
+ assertEquals(encM1_times_encM2, BigInteger.valueOf(5617));
+ assertEquals(encM1plusM2, BigInteger.valueOf(5617));
+
+ logger.info("Successfully completed testPaillierGivenAllParameters: ");
+ }
+
+ @Test
+ public void testPaillierWithKeyGeneration() throws Exception
+ {
+ logger.info("Starting testPaillierWithKeyGeneration: ");
+
+ // Test with and without gmp optimization for modPow
+ SystemConfiguration.setProperty("pallier.FIPSPrimeGenerationChecks", "true");
+ SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
+ SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "true");
+ testPaillerWithKeyGenerationGeneral();
+
+ SystemConfiguration.setProperty("pallier.FIPSPrimeGenerationChecks", "false");
+
+ SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
+ SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "true");
+ testPaillerWithKeyGenerationGeneral();
+
+ SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
+ SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "false");
+ testPaillerWithKeyGenerationGeneral();
+
+ SystemConfiguration.setProperty("paillier.useGMPForModPow", "false");
+ SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "false");
+ testPaillerWithKeyGenerationGeneral();
+
+ // Reset the properties
+ SystemConfiguration.initialize();
+
+ logger.info("Ending testPaillierWithKeyGeneration: ");
+ }
+
+ public void testPaillerWithKeyGenerationGeneral() throws Exception
+ {
+ // Test without requiring highest bit to be set
+ logger.info("Starting testPaillierWithKeyGenerationBitSetOption with ensureHighBitSet = false");
+ testPaillierWithKeyGenerationBitSetOption(-1);
+
+ // Test requiring highest bit to be set
+ logger.info("Starting testPaillierWithKeyGenerationBitSetOption with ensureHighBitSet = true");
+ testPaillierWithKeyGenerationBitSetOption(5);
+ }
+
+ public void testPaillierWithKeyGenerationBitSetOption(int ensureBitSet) throws Exception
+ {
+ Random r = new Random();
+ int lowBitLength = 3073; // inclusive
+ int highBitLength = 7001; // exclusive
+
+ int loopVal = 1; // int loopVal = 1000; //change this and re-test for high loop testing
+ for (int i = 0; i < loopVal; ++i)
+ {
+ logger.info("i = " + i);
+
+ basicTestPaillierWithKeyGeneration(bitLength, certainty, ensureBitSet);
+ basicTestPaillierWithKeyGeneration(3072, certainty, ensureBitSet);
+
+ // Test with random bit length between 3073 and 7000
+ int randomLargeBitLength = r.nextInt(highBitLength - lowBitLength) + lowBitLength;
+ basicTestPaillierWithKeyGeneration(randomLargeBitLength, certainty, ensureBitSet);
+ }
+ }
+
+ private void basicTestPaillierWithKeyGeneration(int bitLengthInput, int certaintyInput, int ensureBitSet) throws Exception
+ {
+ Paillier pailler = new Paillier(bitLengthInput, certaintyInput, ensureBitSet);
+ BigInteger generatedN = pailler.getN();
+ BigInteger geneartedNsquared = generatedN.multiply(generatedN);
+
+ // Check the decrypting the encryption yields the message
+ BigInteger encM1 = pailler.encrypt(m1);
+ BigInteger encM2 = pailler.encrypt(m2);
+ logger.info("encM1 = " + encM1.intValue() + " encM2 = " + encM2.intValue());
+
+ BigInteger decM1 = pailler.decrypt(encM1);
+ BigInteger decM2 = pailler.decrypt(encM2);
+ logger.info("decM1 = " + decM1.intValue() + " decM2 = " + decM2.intValue());
+
+ assertEquals(decM1, m1);
+ assertEquals(decM2, m2);
+
+ // Check homomorphic property: E_r1(m1)*E_r2(m2) mod N^2 = E_r1r2((m1+m2) mod N) mod N^2
+ BigInteger encM1_times_encM2 = (encM1.multiply(encM2)).mod(geneartedNsquared);
+ BigInteger multDecrypt = pailler.decrypt(encM1_times_encM2);
+ BigInteger m1_plus_m2 = (m1.add(m2)).mod(N);
+
+ logger.info("encM1_times_encM2 = " + encM1_times_encM2.intValue() + " multDecrypt = " + multDecrypt.intValue() + " m1_plus_m2 = " + m1_plus_m2.intValue());
+
+ assertEquals(multDecrypt, m1_plus_m2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/general/PartitionUtilsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/general/PartitionUtilsTest.java b/src/test/java/org/apache/pirk/test/general/PartitionUtilsTest.java
new file mode 100644
index 0000000..c1022ad
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/general/PartitionUtilsTest.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.general;
+
+import org.apache.pirk.schema.data.partitioner.IPDataPartitioner;
+import org.apache.pirk.schema.data.partitioner.ISO8601DatePartitioner;
+import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner;
+import org.apache.pirk.utils.PIRException;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigInteger;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Class to functionally test the bit conversion utils
+ */
+public class PartitionUtilsTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(PartitionUtilsTest.class);
+
+ @Test
+ public void testMask()
+ {
+ logger.info("Starting testMask: ");
+
+ assertEquals(0, PrimitiveTypePartitioner.formBitMask(0).intValue());
+
+ assertEquals(0b000000000000001, PrimitiveTypePartitioner.formBitMask(1).intValue());
+ assertEquals(0b000000000001111, PrimitiveTypePartitioner.formBitMask(4).intValue());
+ assertEquals(0b000000001111111, PrimitiveTypePartitioner.formBitMask(7).intValue());
+ assertEquals(0b111111111111111, PrimitiveTypePartitioner.formBitMask(15).intValue());
+
+ assertEquals(new BigInteger("FFFFF", 16), PrimitiveTypePartitioner.formBitMask(20));
+ assertEquals(new BigInteger("FFFFFFFF", 16), PrimitiveTypePartitioner.formBitMask(32));
+ assertEquals(new BigInteger("3FFFFFFFFFF", 16), PrimitiveTypePartitioner.formBitMask(42));
+ assertEquals(new BigInteger("7FFFFFFFFFFFFFFF", 16), PrimitiveTypePartitioner.formBitMask(63));
+
+ logger.info("Successfully completed testMask");
+ }
+
+ @Test
+ public void testPartitionBits() throws PIRException
+ {
+ logger.info("Starting testPartitionBits: ");
+
+ BigInteger value = new BigInteger("245"); // 11110101
+ BigInteger value2 = new BigInteger("983"); // 1111010111
+
+ BigInteger mask4 = PrimitiveTypePartitioner.formBitMask(4); // 1111
+ BigInteger mask8 = PrimitiveTypePartitioner.formBitMask(8); // 11111111
+
+ List<BigInteger> partitions = PrimitiveTypePartitioner.partitionBits(value, 4, mask4);
+ assertEquals(2, partitions.size());
+ assertEquals(0b1111, partitions.get(0).intValue());
+ assertEquals(0b0101, partitions.get(1).intValue());
+
+ partitions = PrimitiveTypePartitioner.partitionBits(value2, 4, mask4);
+ assertEquals(3, partitions.size());
+ assertEquals(0b1111, partitions.get(0).intValue());
+ assertEquals(0b0101, partitions.get(1).intValue());
+ assertEquals(0b0011, partitions.get(2).intValue());
+
+ partitions = PrimitiveTypePartitioner.partitionBits(value, 8, mask8);
+ assertEquals(1, partitions.size());
+ assertEquals(0b11110101, partitions.get(0).intValue());
+
+ try
+ {
+ partitions = PrimitiveTypePartitioner.partitionBits(value, 4, mask8);
+ fail("BitConversionUtils.partitionBits did not throw error for mismatched partitionSize and mask size");
+ } catch (Exception ignore)
+ {
+ // Expected.
+ }
+
+ logger.info("Successfully completed testPartitionBits");
+ }
+
+ @Test
+ public void testPartitions() throws Exception
+ {
+ logger.info("Starting testToPartitions:");
+
+ PrimitiveTypePartitioner primitivePartitioner = new PrimitiveTypePartitioner();
+ IPDataPartitioner ipPartitioner = new IPDataPartitioner();
+ ISO8601DatePartitioner datePartitioner = new ISO8601DatePartitioner();
+
+ // Test IP
+ String ipTest = "127.0.0.1";
+ List<BigInteger> partsIP = ipPartitioner.toPartitions(ipTest, PrimitiveTypePartitioner.STRING);
+ assertEquals(4, partsIP.size());
+ assertEquals(ipTest, ipPartitioner.fromPartitions(partsIP, 0, PrimitiveTypePartitioner.STRING));
+
+ // Test Date
+ String dateTest = "2016-02-20T23:29:05.000Z";
+ List<BigInteger> partsDate = datePartitioner.toPartitions(dateTest, null);
+ assertEquals(8, partsDate.size());
+ assertEquals(dateTest, datePartitioner.fromPartitions(partsDate, 0, null));
+
+ // Test byte
+ byte bTest = Byte.parseByte("10");
+ List<BigInteger> partsByte = primitivePartitioner.toPartitions(bTest, PrimitiveTypePartitioner.BYTE);
+ assertEquals(1, partsByte.size());
+ assertEquals(bTest, primitivePartitioner.fromPartitions(partsByte, 0, PrimitiveTypePartitioner.BYTE));
+
+ partsByte = primitivePartitioner.toPartitions("12", PrimitiveTypePartitioner.BYTE);
+ assertEquals(1, partsByte.size());
+ assertEquals((byte) 12, primitivePartitioner.fromPartitions(partsByte, 0, PrimitiveTypePartitioner.BYTE));
+
+ List<BigInteger> partsByteMax = primitivePartitioner.toPartitions(Byte.MAX_VALUE, PrimitiveTypePartitioner.BYTE);
+ assertEquals(1, partsByteMax.size());
+ assertEquals(Byte.MAX_VALUE, primitivePartitioner.fromPartitions(partsByteMax, 0, PrimitiveTypePartitioner.BYTE));
+
+ // Test string
+ String stringBits = SystemConfiguration.getProperty("pir.stringBits");
+ SystemConfiguration.setProperty("pir.stringBits", "64");
+ testString("testString"); // over the allowed bit size
+ testString("t"); // under the allowed bit size
+ SystemConfiguration.setProperty("pir.stringBits", stringBits);
+
+ // Test short
+ short shortTest = Short.valueOf("2456");
+ List<BigInteger> partsShort = primitivePartitioner.toPartitions(shortTest, PrimitiveTypePartitioner.SHORT);
+ assertEquals(2, partsShort.size());
+ assertEquals(shortTest, primitivePartitioner.fromPartitions(partsShort, 0, PrimitiveTypePartitioner.SHORT));
+
+ partsShort = primitivePartitioner.toPartitions("32767", PrimitiveTypePartitioner.SHORT);
+ assertEquals(2, partsShort.size());
+ assertEquals((short) 32767, primitivePartitioner.fromPartitions(partsShort, 0, PrimitiveTypePartitioner.SHORT));
+
+ partsShort = primitivePartitioner.toPartitions((short) -42, PrimitiveTypePartitioner.SHORT);
+ assertEquals(2, partsShort.size());
+ assertEquals((short) -42, primitivePartitioner.fromPartitions(partsShort, 0, PrimitiveTypePartitioner.SHORT));
+
+ List<BigInteger> partsShortMax = primitivePartitioner.toPartitions(Short.MAX_VALUE, PrimitiveTypePartitioner.SHORT);
+ assertEquals(2, partsShortMax.size());
+ assertEquals(Short.MAX_VALUE, primitivePartitioner.fromPartitions(partsShortMax, 0, PrimitiveTypePartitioner.SHORT));
+
+ // Test int
+ int intTest = Integer.parseInt("-5789");
+ List<BigInteger> partsInt = primitivePartitioner.toPartitions(intTest, PrimitiveTypePartitioner.INT);
+ assertEquals(4, partsInt.size());
+ assertEquals(intTest, primitivePartitioner.fromPartitions(partsInt, 0, PrimitiveTypePartitioner.INT));
+
+ partsInt = primitivePartitioner.toPartitions("2016", PrimitiveTypePartitioner.INT);
+ assertEquals(4, partsInt.size());
+ assertEquals(2016, primitivePartitioner.fromPartitions(partsInt, 0, PrimitiveTypePartitioner.INT));
+
+ partsInt = primitivePartitioner.toPartitions(1386681237, PrimitiveTypePartitioner.INT);
+ assertEquals(4, partsInt.size());
+ assertEquals(1386681237, primitivePartitioner.fromPartitions(partsInt, 0, PrimitiveTypePartitioner.INT));
+
+ List<BigInteger> partsIntMax = primitivePartitioner.toPartitions(Integer.MAX_VALUE, PrimitiveTypePartitioner.INT);
+ assertEquals(4, partsIntMax.size());
+ assertEquals(Integer.MAX_VALUE, primitivePartitioner.fromPartitions(partsIntMax, 0, PrimitiveTypePartitioner.INT));
+
+ // Test long
+ long longTest = Long.parseLong("56789");
+ List<BigInteger> partsLong = primitivePartitioner.toPartitions(longTest, PrimitiveTypePartitioner.LONG);
+ assertEquals(8, partsLong.size());
+ assertEquals(longTest, primitivePartitioner.fromPartitions(partsLong, 0, PrimitiveTypePartitioner.LONG));
+
+ List<BigInteger> partsLongMax = primitivePartitioner.toPartitions(Long.MAX_VALUE, PrimitiveTypePartitioner.LONG);
+ assertEquals(8, partsLongMax.size());
+ assertEquals(Long.MAX_VALUE, primitivePartitioner.fromPartitions(partsLongMax, 0, PrimitiveTypePartitioner.LONG));
+
+ // Test float
+ float floatTest = Float.parseFloat("567.77");
+ List<BigInteger> partsFloat = primitivePartitioner.toPartitions(floatTest, PrimitiveTypePartitioner.FLOAT);
+ assertEquals(4, partsFloat.size());
+ assertEquals(floatTest, primitivePartitioner.fromPartitions(partsFloat, 0, PrimitiveTypePartitioner.FLOAT));
+
+ partsFloat = primitivePartitioner.toPartitions(-99.99f, PrimitiveTypePartitioner.FLOAT);
+ assertEquals(4, partsFloat.size());
+ assertEquals(-99.99f, primitivePartitioner.fromPartitions(partsFloat, 0, PrimitiveTypePartitioner.FLOAT));
+
+ List<BigInteger> partsFloatMax = primitivePartitioner.toPartitions(Float.MAX_VALUE, PrimitiveTypePartitioner.FLOAT);
+ assertEquals(4, partsFloatMax.size());
+ assertEquals(Float.MAX_VALUE, primitivePartitioner.fromPartitions(partsFloatMax, 0, PrimitiveTypePartitioner.FLOAT));
+
+ // Test double
+ double doubleTest = Double.parseDouble("567.77");
+ List<BigInteger> partsDouble = primitivePartitioner.toPartitions(doubleTest, PrimitiveTypePartitioner.DOUBLE);
+ assertEquals(8, partsDouble.size());
+ assertEquals(doubleTest, primitivePartitioner.fromPartitions(partsDouble, 0, PrimitiveTypePartitioner.DOUBLE));
+
+ List<BigInteger> partsDoubleMax = primitivePartitioner.toPartitions(Double.MAX_VALUE, PrimitiveTypePartitioner.DOUBLE);
+ assertEquals(8, partsDoubleMax.size());
+ assertEquals(Double.MAX_VALUE, primitivePartitioner.fromPartitions(partsDoubleMax, 0, PrimitiveTypePartitioner.DOUBLE));
+
+ // Test char
+ char charTest = 'b';
+ List<BigInteger> partsChar = primitivePartitioner.toPartitions(charTest, PrimitiveTypePartitioner.CHAR);
+ assertEquals(2, partsChar.size());
+ assertEquals(charTest, primitivePartitioner.fromPartitions(partsChar, 0, PrimitiveTypePartitioner.CHAR));
+
+ // Ensure Endianness preserved
+ charTest = '\uFFFE';
+ partsChar = primitivePartitioner.toPartitions(charTest, PrimitiveTypePartitioner.CHAR);
+ assertEquals(2, partsChar.size());
+ assertEquals(charTest, primitivePartitioner.fromPartitions(partsChar, 0, PrimitiveTypePartitioner.CHAR));
+
+ charTest = '\uFEFF';
+ partsChar = primitivePartitioner.toPartitions(charTest, PrimitiveTypePartitioner.CHAR);
+ assertEquals(2, partsChar.size());
+ assertEquals(charTest, primitivePartitioner.fromPartitions(partsChar, 0, PrimitiveTypePartitioner.CHAR));
+
+ List<BigInteger> partsCharMax = primitivePartitioner.toPartitions(Character.MAX_VALUE, PrimitiveTypePartitioner.CHAR);
+ assertEquals(2, partsCharMax.size());
+ assertEquals(Character.MAX_VALUE, primitivePartitioner.fromPartitions(partsCharMax, 0, PrimitiveTypePartitioner.CHAR));
+
+ logger.info("Sucessfully completed testToPartitions:");
+ }
+
+ @Test
+ public void testPaddedPartitions() throws PIRException
+ {
+ PrimitiveTypePartitioner primitivePartitioner = new PrimitiveTypePartitioner();
+
+ List<String> primitiveTypes = Arrays.asList(PrimitiveTypePartitioner.BYTE, PrimitiveTypePartitioner.CHAR, PrimitiveTypePartitioner.SHORT,
+ PrimitiveTypePartitioner.INT, PrimitiveTypePartitioner.LONG, PrimitiveTypePartitioner.FLOAT, PrimitiveTypePartitioner.DOUBLE,
+ PrimitiveTypePartitioner.STRING);
+ for (String type : primitiveTypes)
+ {
+ assertEquals(primitivePartitioner.getNumPartitions(type), primitivePartitioner.getPaddedPartitions(type).size());
+ }
+ }
+
+ private void testString(String testString) throws Exception
+ {
+ PrimitiveTypePartitioner ptp = new PrimitiveTypePartitioner();
+
+ List<BigInteger> partsString = ptp.toPartitions(testString, PrimitiveTypePartitioner.STRING);
+ int numParts = Integer.parseInt(SystemConfiguration.getProperty("pir.stringBits")) / 8;
+ assertEquals(numParts, partsString.size());
+
+ logger.info("testString.getBytes().length = " + testString.getBytes().length);
+ int offset = numParts;
+ if (testString.getBytes().length < numParts)
+ {
+ offset = testString.getBytes().length;
+ }
+ String element = new String(testString.getBytes(), 0, offset);
+ assertEquals(element, ptp.fromPartitions(partsString, 0, PrimitiveTypePartitioner.STRING));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/general/QueryParserUtilsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/general/QueryParserUtilsTest.java b/src/test/java/org/apache/pirk/test/general/QueryParserUtilsTest.java
new file mode 100644
index 0000000..b3db140
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/general/QueryParserUtilsTest.java
@@ -0,0 +1,421 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.general;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.apache.hadoop.io.MapWritable;
+import org.apache.pirk.schema.data.DataSchema;
+import org.apache.pirk.schema.data.DataSchemaRegistry;
+import org.apache.pirk.schema.query.QuerySchemaRegistry;
+import org.apache.pirk.test.utils.Inputs;
+import org.apache.pirk.utils.QueryParserUtils;
+import org.apache.pirk.utils.StringUtils;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.json.simple.JSONObject;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class for testing the QueryParser methods
+ */
+public class QueryParserUtilsTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(QueryParserUtilsTest.class);
+
+ private static MapWritable doc = null; // MapWritable with arrays in json string representation
+ private static MapWritable docWAW = null; // MapWritable with arrays as WritableArrayWritable objects
+ private static Map<String,Object> docMap = null; // arrays as ArrayList<String>
+
+ private static DataSchema dSchema = null;
+
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ ArrayList<JSONObject> dataElementsJSON = Inputs.createJSONDataElements();
+
+ // Reset the schema properties and registries
+ DataSchemaRegistry.clearRegistry();
+ QuerySchemaRegistry.clearRegistry();
+ SystemConfiguration.setProperty("data.schemas", "none");
+ SystemConfiguration.setProperty("query.schemas", "none");
+
+ Inputs.createSchemaFiles(null, false, null);
+
+ dSchema = DataSchemaRegistry.get(Inputs.TEST_DATA_SCHEMA_NAME);
+
+ // ProcessBuilder pAdd1 = new ProcessBuilder("curl", "-XPUT", indexTypeNum1, "-d",
+ // "{\"qname\":\"a.b.c.com\",\"date\":\"2016-02-20T23:29:05.000Z\",\"qtype\":[\"1\"]"
+ // + ",\"rcode\":\"0\",\"src_ip\":\"55.55.55.55\",\"dest_ip\":\"1.2.3.6\"" + ",\"ip\":[\"10.20.30.40\",\"10.20.30.60\"]}");
+ //
+ doc = StringUtils.jsonStringToMapWritableWithArrayWritable(dataElementsJSON.get(0).toJSONString(), dSchema);
+ docWAW = StringUtils.jsonStringToMapWritableWithWritableArrayWritable(dataElementsJSON.get(0).toJSONString(), dSchema);
+ docMap = StringUtils.jsonStringToMap(dataElementsJSON.get(0).toJSONString(), dSchema);
+ }
+
+ @AfterClass
+ public static void teardown()
+ {
+ // Reset the schema properties and registries
+ DataSchemaRegistry.clearRegistry();
+ QuerySchemaRegistry.clearRegistry();
+ SystemConfiguration.setProperty("data.schemas", "none");
+ SystemConfiguration.setProperty("query.schemas", "none");
+ }
+
+ @Test
+ public void testSingleQuery()
+ {
+ String query1 = "?q=src_ip:55.55.55.55";
+ assertTrue(QueryParserUtils.checkRecord(query1, doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query1, docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecord(query1, docMap, dSchema));
+
+ String query2 = "?q=qname:a.b.c.com";
+ assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema));
+
+ String query3 = "?q=qname:d.b.c.com";
+ assertFalse(QueryParserUtils.checkRecord(query3, doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecord(query3, docMap, dSchema));
+ }
+
+ @Test
+ public void testQueryFieldDoesNotExist()
+ {
+ logger.info("running testQueryFieldDoesNotExist");
+
+ // Field does not exist, this should not be found
+ String query = "?q=nonexistent-field:*check*";
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query, docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecord(query, doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord(query, docMap, dSchema));
+
+ // First field does not exist, but second should be found
+ String query2 = "?q=nonexistent-field:*check*+OR+qname:*a.b.c.com*";
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema));
+
+ // First field does not exist, second field does, but AND operator makes query false
+ String query3 = "?q=nonexistent-field:*check*+AND+qname:*a.b.c.com*";
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecord(query3, doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord(query3, docMap, dSchema));
+
+ logger.info("completed testQueryFieldDoesNotExist");
+ }
+
+ @Test
+ public void testIgnoreCase()
+ {
+ logger.info("running testIgnoreCase");
+
+ // with case sensitivity, should NOT be found
+ String query = "?q=qname:*A.b.c.com*";
+ assertFalse(QueryParserUtils.checkRecord(query, doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query, docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecord(query, docMap, dSchema));
+
+ // with case sensitivity, should be found
+ String query2 = "?q=qname:*a.b.c.com*";
+ assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema));
+
+ // adds @ flag = case insensitivity, thus should be found
+ String query3 = "?q=qname@:*A.b.c.com*";
+ assertTrue(QueryParserUtils.checkRecord(query3, doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecord(query3, docMap, dSchema));
+
+ logger.info("completed testIgnoreCase");
+ }
+
+ @Test
+ public void testSingleValueRangeQuery()
+ {
+ testSingleValueRangeQueryMapWritable();
+ testSingleValueRangeQueryMap();
+ testSingleValueRangeQueryMapWritableWAW();
+ }
+
+ private void testSingleValueRangeQueryMapWritable()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=rcode:[0+TO+2]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=rcode:{-1+TO+2}", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=rcode:[-1+TO+0]", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=rcode:{0+TO+3}", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=rcode:[3+TO+10]", doc, dSchema));
+ }
+
+ private void testSingleValueRangeQueryMap()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=rcode:[0+TO+2]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=rcode:{-1+TO+2}", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=rcode:[-1+TO+0]", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=rcode:{0+TO+3}", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=rcode:[3+TO+10]", docMap, dSchema));
+ }
+
+ private void testSingleValueRangeQueryMapWritableWAW()
+ {
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[0+TO+2]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:{-1+TO+2}", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[-1+TO+0]", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:{0+TO+3}", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[3+TO+10]", docWAW, dSchema));
+ }
+
+ @Test
+ public void testIPRangeQuery()
+ {
+ testIPRangeQueryMapWritable();
+ testIPRangeQueryMap();
+ testIPRangeQueryMapWritableWAW();
+ }
+
+ public void testIPRangeQueryMapWritable()
+ {
+ // src_ip: 55.55.55.55
+ // ip: 10.20.30.40,10.20.30.60
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", doc, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", doc, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=ip:[10.20.30.50+TO+10.20.30.69]", doc, dSchema));
+ }
+
+ public void testIPRangeQueryMapWritableWAW()
+ {
+ // src_ip: 55.55.55.55
+ // ip: 10.20.30.40,10.20.30.60
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", docWAW, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", docWAW, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=ip:[10.20.30.50+TO+10.20.30.69]", docWAW, dSchema));
+ }
+
+ public void testIPRangeQueryMap()
+ {
+ // src_ip: 55.55.55.55
+ // ip: 10.20.30.40,10.20.30.60
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", docMap, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", docMap, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=ip:[10.20.30.50+TO+10.20.30.69]", docMap, dSchema));
+ }
+
+ @Test
+ public void testDateRangeQuery()
+ {
+ testDateRangeQueryMapWritable();
+ testDateRangeQueryMapWritableWAW();
+ testDateRangeQueryMap();
+ }
+
+ private void testDateRangeQueryMapWritable()
+ {
+ // date: 2016-02-20T23:29:05.000Z
+
+ assertTrue(QueryParserUtils.checkRecord("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", doc, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", doc, dSchema));
+ }
+
+ private void testDateRangeQueryMap()
+ {
+ // date: 2016-02-20T23:29:05.000Z
+
+ assertTrue(QueryParserUtils.checkRecord("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", docMap, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", docMap, dSchema));
+ }
+
+ private void testDateRangeQueryMapWritableWAW()
+ {
+ // date: 2016-02-20T23:29:05.000Z
+
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", docWAW, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", docWAW, dSchema));
+ }
+
+ @Test
+ public void testBooleanQuery()
+ {
+ testBooleanQueryMapWritable();
+ testBooleanQueryMapMapWritableWAW();
+ testBooleanQueryMap();
+ }
+
+ private void testBooleanQueryMapWritable()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+qtype:2+OR+rcode:0", doc, dSchema));
+ }
+
+ private void testBooleanQueryMap()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+qtype:2+OR+rcode:0", docMap, dSchema));
+ }
+
+ private void testBooleanQueryMapMapWritableWAW()
+ {
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]",
+ docWAW, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]",
+ docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]",
+ docWAW, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:5+OR+qtype:2+OR+rcode:0", docWAW, dSchema));
+ }
+
+ @Test
+ public void testAllQuery()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=*", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=*", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=*", docWAW, dSchema));
+ }
+
+ @Test
+ public void testWildcardQuery()
+ {
+ testWildcardQueryMapWritable();
+ testWildcardQueryMap();
+ testWildcardQueryMapWritableWAW();
+ }
+
+ private void testWildcardQueryMapWritable()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:*.com", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c*m", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b*", doc, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:*.org", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:mrtf*", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljnik*.uk", doc, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c?m", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.?.com", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:?.b.c.com", doc, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:medelj?ikafera.com", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljntkafer?.com", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:?edeljnikrfera.com", doc, dSchema));
+ }
+
+ private void testWildcardQueryMap()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:*.com", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c*m", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b*", docMap, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:*.org", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:mrtf*", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljnik*.uk", docMap, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c?m", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.?.com", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:?.b.c.com", docMap, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:medelj?ikafera.com", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljntkafer?.com", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:?edeljnikrfera.com", docMap, dSchema));
+ }
+
+ private void testWildcardQueryMapWritableWAW()
+ {
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:*.com", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.c.c*m", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b*", docWAW, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:*.org", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:mrtf*", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:nedeljnik*.uk", docWAW, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.c.c?m", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.?.com", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:?.b.c.com", docWAW, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:medelj?ikafera.com", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:nedeljntkafer?.com", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:?edeljnikrfera.com", docWAW, dSchema));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/schema/data/LoadDataSchemaTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/schema/data/LoadDataSchemaTest.java b/src/test/java/org/apache/pirk/test/schema/data/LoadDataSchemaTest.java
new file mode 100644
index 0000000..9f52af3
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/schema/data/LoadDataSchemaTest.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.schema.data;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.apache.pirk.schema.data.DataSchema;
+import org.apache.pirk.schema.data.DataSchemaLoader;
+import org.apache.pirk.schema.data.DataSchemaRegistry;
+import org.apache.pirk.schema.data.partitioner.IPDataPartitioner;
+import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner;
+import org.apache.pirk.test.utils.TestUtils;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+/**
+ * Test suite for LoadDataSchema and DataSchema
+ */
+public class LoadDataSchemaTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(LoadDataSchemaTest.class);
+
+ private String dataSchemaName = "fakeDataSchema";
+
+ private String element1 = "elementName1";
+ private String element2 = "elementName2";
+ private String element3 = "elementName3";
+
+ @Test
+ public void testGeneralSchemaLoad() throws Exception
+ {
+ // Pull off the property and reset upon completion
+ String schemasProp = SystemConfiguration.getProperty("data.schemas", "none");
+
+ // Write the schema file
+ try
+ {
+ createDataSchema("schemaFile");
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+
+ // Force the schema to load
+ DataSchemaLoader.initialize();
+
+ // Check the entries
+ DataSchema dSchema = DataSchemaRegistry.get(dataSchemaName);
+
+ assertEquals(dataSchemaName, dSchema.getSchemaName());
+
+ assertEquals(3, dSchema.getElementNames().size());
+
+ // TODO: check Hadoop text names
+
+ assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element1));
+ assertEquals(PrimitiveTypePartitioner.INT, dSchema.getElementType(element2));
+ assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element3));
+
+ assertEquals(PrimitiveTypePartitioner.class.getName(), dSchema.getPartitionerTypeName(element1));
+ if (!(dSchema.getPartitionerForElement(element1) instanceof PrimitiveTypePartitioner))
+ {
+ fail("Partitioner instance for element1 must be PrimitiveTypePartitioner");
+ }
+ assertEquals(IPDataPartitioner.class.getName(), dSchema.getPartitionerTypeName(element3));
+ if (!(dSchema.getPartitionerForElement(element3) instanceof IPDataPartitioner))
+ {
+ fail("Partitioner instance for element3 must be IPDataPartitioner");
+ }
+
+ assertEquals(2, dSchema.getArrayElements().size());
+ assertTrue(dSchema.getArrayElements().contains(element2));
+ assertTrue(dSchema.getArrayElements().contains(element3));
+
+ assertEquals(1, dSchema.getNonArrayElements().size());
+ assertTrue(dSchema.getNonArrayElements().contains(element1));
+
+ // Reset original data.schemas property
+ SystemConfiguration.setProperty("data.schemas", schemasProp);
+
+ // Force the schema to load
+ if (!schemasProp.equals("none"))
+ {
+ DataSchemaLoader.initialize();
+ }
+ }
+
+ @Test
+ public void testIncorrectJavaType() throws Exception
+ {
+ // Pull off the property and reset upon completion
+ String schemasProp = SystemConfiguration.getProperty("data.schemas");
+
+ // Write the schema file
+ try
+ {
+ createDataSchemaIncorrectJavaType("wrongJavaType");
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+
+ try
+ {
+ // Force the schema to load
+ DataSchemaLoader.initialize();
+ fail("DataSchemaLoader did not throw exception for incorrect javaType");
+ } catch (Exception ignore)
+ {}
+
+ // Reset original data.schemas property
+ SystemConfiguration.setProperty("data.schemas", schemasProp);
+
+ // Force the schema to load
+ DataSchemaLoader.initialize();
+ }
+
+ @Test
+ public void testUnknownPartitioner() throws Exception
+ {
+ // Pull off the property and reset upon completion
+ String schemasProp = SystemConfiguration.getProperty("data.schemas");
+
+ // Write the schema file
+ try
+ {
+ createDataSchemaUnknownPartitioner("unknownPartitioner");
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+
+ try
+ {
+ // Force the schema to load
+ DataSchemaLoader.initialize();
+ fail("DataSchemaLoader did not throw exception for unknown partitioner");
+ } catch (Exception ignore)
+ {}
+
+ // Reset original data.schemas property
+ SystemConfiguration.setProperty("data.schemas", schemasProp);
+
+ // Force the schema to load
+ DataSchemaLoader.initialize();
+ }
+
+ // Create the file that contains an unknown partitioner
+ private void createDataSchemaUnknownPartitioner(String schemaFile) throws IOException
+ {
+ // Create a temporary file for the test schema, set in the properties
+ File file = File.createTempFile(schemaFile, ".xml");
+ file.deleteOnExit();
+ logger.info("file = " + file.toString());
+ SystemConfiguration.setProperty("data.schemas", file.toString());
+
+ // Write to the file
+ try
+ {
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+ Document doc = dBuilder.newDocument();
+
+ // root element
+ Element rootElement = doc.createElement("schema");
+ doc.appendChild(rootElement);
+
+ // Add the schemaName
+ Element schemaNameElement = doc.createElement("schemaName");
+ schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
+ rootElement.appendChild(schemaNameElement);
+
+ // Add the element - unknown partitioner
+ TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.INT, "false", "fakePartitioner");
+
+ // Write to a xml file
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ Transformer transformer = transformerFactory.newTransformer();
+ DOMSource source = new DOMSource(doc);
+ StreamResult result = new StreamResult(file);
+ transformer.transform(source, result);
+
+ // Output for testing
+ StreamResult consoleResult = new StreamResult(System.out);
+ transformer.transform(source, consoleResult);
+ System.out.println();
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ }
+ }
+
+ // Create the test data schema file
+ private void createDataSchema(String schemaFile) throws IOException
+ {
+ // Create a temporary file for the test schema, set in the properties
+ File file = File.createTempFile(schemaFile, ".xml");
+ file.deleteOnExit();
+ logger.info("file = " + file.toString());
+ SystemConfiguration.setProperty("data.schemas", file.toString());
+
+ // Write to the file
+ try
+ {
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+ Document doc = dBuilder.newDocument();
+
+ // root element
+ Element rootElement = doc.createElement("schema");
+ doc.appendChild(rootElement);
+
+ // Add the schemaName
+ Element schemaNameElement = doc.createElement("schemaName");
+ schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
+ rootElement.appendChild(schemaNameElement);
+
+ // Add the elements
+ // element1 -- single String
+ // TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.STRING, "false", PrimitiveTypePartitioner.class.getName());
+ TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.STRING, "false", null);
+
+ // element2 - -- array of Integers
+ TestUtils.addElement(doc, rootElement, element2, PrimitiveTypePartitioner.INT, "true", PrimitiveTypePartitioner.class.getName());
+
+ // element3 -- array of IP addresses
+ TestUtils.addElement(doc, rootElement, element3, PrimitiveTypePartitioner.STRING, "true", IPDataPartitioner.class.getName());
+
+ // Write to a xml file
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ Transformer transformer = transformerFactory.newTransformer();
+ DOMSource source = new DOMSource(doc);
+ StreamResult result = new StreamResult(file);
+ transformer.transform(source, result);
+
+ // Output for testing
+ StreamResult consoleResult = new StreamResult(System.out);
+ transformer.transform(source, consoleResult);
+ System.out.println();
+
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ }
+ }
+
+ // Create the test schema file
+ private void createDataSchemaIncorrectJavaType(String schemaFile) throws IOException
+ {
+ // Create a temporary file for the test schema, set in the properties
+ File file = File.createTempFile(schemaFile, ".xml");
+ file.deleteOnExit();
+ logger.info("file = " + file.toString());
+ SystemConfiguration.setProperty("data.schemas", file.toString());
+
+ // Write to the file
+ try
+ {
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+ Document doc = dBuilder.newDocument();
+
+ // root element
+ Element rootElement = doc.createElement("schema");
+ doc.appendChild(rootElement);
+
+ // Add the schemaName
+ Element schemaNameElement = doc.createElement("schemaName");
+ schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
+ rootElement.appendChild(schemaNameElement);
+
+ // Add the element - unknown Java type
+ TestUtils.addElement(doc, rootElement, element1, "bogus", "false", PrimitiveTypePartitioner.class.getName());
+
+ // Write to a xml file
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ Transformer transformer = transformerFactory.newTransformer();
+ DOMSource source = new DOMSource(doc);
+ StreamResult result = new StreamResult(file);
+ transformer.transform(source, result);
+
+ // Output for testing
+ StreamResult consoleResult = new StreamResult(System.out);
+ transformer.transform(source, consoleResult);
+ System.out.println();
+
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ }
+ }
+}
[6/6] incubator-pirk git commit: Move all tests under src/test
directory, this closes apache/incubator-pirk#66
Posted by sm...@apache.org.
Move all tests under src/test directory, this closes apache/incubator-pirk#66
Project: http://git-wip-us.apache.org/repos/asf/incubator-pirk/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-pirk/commit/a643ae68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-pirk/tree/a643ae68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-pirk/diff/a643ae68
Branch: refs/heads/master
Commit: a643ae689a18b995dc4256a8712875698b3e92af
Parents: 8a1c4e1
Author: Tim Ellison <t....@gmail.com>
Authored: Thu Aug 18 11:31:38 2016 -0400
Committer: smarthi <sm...@apache.org>
Committed: Thu Aug 18 11:31:38 2016 -0400
----------------------------------------------------------------------
pom.xml | 2 -
.../apache/pirk/benchmark/BenchmarkDriver.java | 36 --
.../pirk/benchmark/PaillierBenchmark.java | 126 ----
.../test/distributed/DistributedTestCLI.java | 188 ------
.../test/distributed/DistributedTestDriver.java | 149 -----
.../distributed/testsuite/DistTestSuite.java | 458 -------------
.../org/apache/pirk/test/utils/BaseTests.java | 643 -------------------
.../java/org/apache/pirk/test/utils/Inputs.java | 606 -----------------
.../apache/pirk/test/utils/StandaloneQuery.java | 164 -----
.../org/apache/pirk/test/utils/TestUtils.java | 312 ---------
.../pirk/general/ISO8601DateParserTest.java | 50 --
.../org/apache/pirk/general/KeyedHashTest.java | 83 ---
.../org/apache/pirk/general/PaillierTest.java | 303 ---------
.../apache/pirk/general/PartitionUtilsTest.java | 269 --------
.../pirk/general/QueryParserUtilsTest.java | 421 ------------
.../pirk/schema/data/LoadDataSchemaTest.java | 324 ----------
.../pirk/schema/query/LoadQuerySchemaTest.java | 368 -----------
.../pirk/serialization/SerializationTest.java | 134 ----
.../pirk/test/benchmark/BenchmarkDriver.java | 36 ++
.../pirk/test/benchmark/PaillierBenchmark.java | 126 ++++
.../test/distributed/DistributedTestCLI.java | 188 ++++++
.../test/distributed/DistributedTestDriver.java | 149 +++++
.../distributed/testsuite/DistTestSuite.java | 458 +++++++++++++
.../test/general/ISO8601DateParserTest.java | 50 ++
.../apache/pirk/test/general/KeyedHashTest.java | 83 +++
.../apache/pirk/test/general/PaillierTest.java | 303 +++++++++
.../pirk/test/general/PartitionUtilsTest.java | 269 ++++++++
.../pirk/test/general/QueryParserUtilsTest.java | 421 ++++++++++++
.../test/schema/data/LoadDataSchemaTest.java | 327 ++++++++++
.../test/schema/query/LoadQuerySchemaTest.java | 371 +++++++++++
.../test/serialization/SerializationTest.java | 137 ++++
.../org/apache/pirk/test/utils/BaseTests.java | 643 +++++++++++++++++++
.../java/org/apache/pirk/test/utils/Inputs.java | 606 +++++++++++++++++
.../apache/pirk/test/utils/StandaloneQuery.java | 164 +++++
.../org/apache/pirk/test/utils/TestUtils.java | 312 +++++++++
.../wideskies/standalone/StandaloneTest.java | 128 ++++
.../wideskies/standalone/StandaloneTest.java | 128 ----
37 files changed, 4771 insertions(+), 4764 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ff2eb3f..5a68e32 100644
--- a/pom.xml
+++ b/pom.xml
@@ -451,7 +451,6 @@
<version>3.0.1</version>
<configuration>
<excludes>
- <exclude>org/apache/pirk/benchmark/**</exclude>
<exclude>*/openjdk/**</exclude>
<exclude>generated-sources/**</exclude>
</excludes>
@@ -464,7 +463,6 @@
<version>3.0.1</version>
<configuration>
<excludes>
- <exclude>org/apache/pirk/benchmark/**</exclude>
<exclude>*/openjdk/**</exclude>
<exclude>generated-sources/**</exclude>
</excludes>
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/main/java/org/apache/pirk/benchmark/BenchmarkDriver.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/benchmark/BenchmarkDriver.java b/src/main/java/org/apache/pirk/benchmark/BenchmarkDriver.java
deleted file mode 100644
index a24d0da..0000000
--- a/src/main/java/org/apache/pirk/benchmark/BenchmarkDriver.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pirk.benchmark;
-
-import java.io.IOException;
-
-import org.openjdk.jmh.Main;
-import org.openjdk.jmh.runner.RunnerException;
-
-/**
- * Driver for JMH benchmarking
- */
-public class BenchmarkDriver
-{
- public static void main(String[] args) throws RunnerException, IOException
- {
- Main.main(args);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/main/java/org/apache/pirk/benchmark/PaillierBenchmark.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/benchmark/PaillierBenchmark.java b/src/main/java/org/apache/pirk/benchmark/PaillierBenchmark.java
deleted file mode 100644
index 95f850d..0000000
--- a/src/main/java/org/apache/pirk/benchmark/PaillierBenchmark.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pirk.benchmark;
-
-import java.math.BigInteger;
-
-import org.apache.pirk.encryption.ModPowAbstraction;
-import org.apache.pirk.encryption.Paillier;
-import org.apache.pirk.utils.PIRException;
-import org.apache.pirk.utils.SystemConfiguration;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A JMH benchmark to evaluate Paillier performance both with and without using com.square.jnagmp.gmp to accelerate modPow
- * <p>
- * Guides to using JMH can be found at: http://tutorials.jenkov.com/java-performance/jmh.html and http://nitschinger.at/Using-JMH-for-Java-Microbenchmarking/
- */
-
-public class PaillierBenchmark
-{
- private static final int MODULUS_SIZE = 3074;
- private static final Logger logger = LoggerFactory.getLogger(PaillierBenchmark.class);
-
- @State(Scope.Benchmark)
- public static class PaillierBenchmarkState
- {
- BigInteger r1 = null; // random number in (Z/NZ)*
- BigInteger m1 = null; // message to encrypt
-
- Paillier pallier = null;
-
- /**
- * This sets up the state for the two separate benchmarks
- */
- @Setup(org.openjdk.jmh.annotations.Level.Trial)
- public void setUp()
- {
- int systemPrimeCertainty = SystemConfiguration.getIntProperty("pir.primeCertainty", 100);
- try
- {
- pallier = new Paillier(MODULUS_SIZE, systemPrimeCertainty);
-
- } catch (PIRException e)
- {
- System.out.printf("Couldn't build pallier object!%n");
- }
-
- r1 = BigInteger.valueOf(3);
- m1 = BigInteger.valueOf(5);
- }
- }
-
- @Benchmark
- @BenchmarkMode(Mode.Throughput)
- public void testWithGMP(PaillierBenchmarkState allState)
- {
- SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
- SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "false");
- ModPowAbstraction.reloadConfiguration();
-
- try
- {
- allState.pallier.encrypt(allState.m1, allState.r1);
- } catch (PIRException e)
- {
- logger.info("Exception in testWithGMP!\n");
- }
- }
-
- @Benchmark
- @BenchmarkMode(Mode.Throughput)
- public void testWithGMPConstantTime(PaillierBenchmarkState allState)
- {
- SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
- SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "true");
- ModPowAbstraction.reloadConfiguration();
-
- try
- {
- allState.pallier.encrypt(allState.m1, allState.r1);
- } catch (PIRException e)
- {
- logger.info("Exception in testWithGMPConstantTime!\n");
- }
- }
-
- @Benchmark
- @BenchmarkMode(Mode.Throughput)
- public void testWithoutGMP(PaillierBenchmarkState allState)
- {
- SystemConfiguration.setProperty("paillier.useGMPForModPow", "false");
- ModPowAbstraction.reloadConfiguration();
-
- try
- {
- allState.pallier.encrypt(allState.m1, allState.r1);
- } catch (PIRException e)
- {
- logger.info("Exception in testWithoutGMP!\n");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/main/java/org/apache/pirk/test/distributed/DistributedTestCLI.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/test/distributed/DistributedTestCLI.java b/src/main/java/org/apache/pirk/test/distributed/DistributedTestCLI.java
deleted file mode 100644
index 1535e1f..0000000
--- a/src/main/java/org/apache/pirk/test/distributed/DistributedTestCLI.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.test.distributed;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A container for Apache's Command Line Interface that contains custom functionality for the MapReduce functional tests.
- */
-public class DistributedTestCLI
-{
- private static final Logger logger = LoggerFactory.getLogger(DistributedTestCLI.class);
-
- private CommandLine commandLine = null;
- private Options cliOptions = null;
-
- /**
- * Create and parse allowable options
- *
- * @param args
- * - arguments fed into the main method
- */
- public DistributedTestCLI(String[] args)
- {
- // create the command line options
- cliOptions = createOptions();
-
- try
- {
- // parse the command line options
- CommandLineParser parser = new GnuParser();
- commandLine = parser.parse(cliOptions, args, true);
-
- // if help option is selected, just print help text and exit
- if (hasOption("h"))
- {
- printHelp();
- System.exit(1);
- }
-
- // The full path of the jar file must be set
- if (!hasOption("j"))
- {
- logger.info("The full path of the jar file must be set with -j");
- System.exit(1);
- }
- } catch (Exception e)
- {
- e.printStackTrace();
- System.exit(1);
- }
- }
-
- /**
- * Determine if an option was provided by the user via the CLI
- *
- * @param option
- * - the option of interest
- * @return true if option was provided, false otherwise
- */
- public boolean hasOption(String option)
- {
- return commandLine.hasOption(option);
- }
-
- /**
- * Obtain the argument of the option provided by the user via the CLI
- *
- * @param option
- * - the option of interest
- * @return value of the argument of the option
- */
- public String getOptionValue(String option)
- {
- return commandLine.getOptionValue(option);
- }
-
- /**
- * Determine if the argument was provided, which determines if a test should or should not be run
- *
- * @param allowed
- * - argument string you are looking for
- * @return true if argument was provided via the CLI, false otherwise
- */
- public boolean run(String allowed)
- {
- return run(allowed, "t");
- }
-
- /**
- * Determine if the argument was provided for the selected option, which determines if a test should or should not be run
- *
- * @param allowed
- * - argument string you are looking for
- * @param option
- * - the option of interest
- * @return true if argument was provided via the CLI, false otherwise
- */
- public boolean run(String allowed, String option)
- {
- if (!hasOption(option))
- {
- return true;
- }
-
- String selection = getOptionValue(option);
- String[] selectionList = selection.split(",");
-
- for (String selectionItem : selectionList)
- {
- if (selectionItem.equals(allowed))
- {
- return true;
- }
- }
-
- return false;
- }
-
- /**
- * Create the options available for the DistributedTestDriver
- *
- * @return Apache's CLI Options object
- */
- private Options createOptions()
- {
- Options options = new Options();
-
- // help
- Option optionHelp = new Option("h", "help", false, "Print out the help documentation for this command line execution");
- optionHelp.setRequired(false);
- options.addOption(optionHelp);
-
- // jar file
- Option optionJar = new Option("j", "jar", true, "required -- Fully qualified jar file");
- optionJar.setRequired(false);
- options.addOption(optionJar);
-
- // test selection
- String tests = "testNum = 1: Wideskies Tests\n";
- tests += "Subtests:\n";
- tests += "E - Elasticsearch MapReduce\n";
- tests += "J - JSON/HDFS MapReduce\n";
- tests += "ES - Elasticsearch Spark \n";
- tests += "JS - JSON/HDFS Spark \n";
-
- Option optionTestSelection = new Option("t", "tests", true, "optional -- Select which tests to execute: \n" + tests);
- optionTestSelection.setRequired(false);
- optionTestSelection.setArgName("<testNum>:<subtestDesignator>");
- optionTestSelection.setType(String.class);
- options.addOption(optionTestSelection);
-
- return options;
- }
-
- /**
- * Prints out the help message
- */
- private void printHelp()
- {
- HelpFormatter formatter = new HelpFormatter();
- formatter.setWidth(140);
- formatter.printHelp("DistributedTestDriver", cliOptions);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java b/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java
deleted file mode 100755
index ee37e63..0000000
--- a/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.test.distributed;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.pirk.schema.data.DataSchemaLoader;
-import org.apache.pirk.schema.query.QuerySchemaLoader;
-import org.apache.pirk.schema.query.filter.StopListFilter;
-import org.apache.pirk.test.distributed.testsuite.DistTestSuite;
-import org.apache.pirk.test.utils.Inputs;
-import org.apache.pirk.utils.SystemConfiguration;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Driver class to run the suite of functional tests for MR and Spark PIR
- *
- */
-public class DistributedTestDriver
-{
- private static final Logger logger = LoggerFactory.getLogger(DistributedTestDriver.class);
-
- // Input
- public static final String JSON_PIR_INPUT_FILE_PROPERTY = "test.pir.inputJSONFile";
- public static final String ES_PIR_INPUT_INDEX_PROPERTY = "test.pir.es.index";
- public static final String PIR_QUERY_INPUT_DIR = "test.pir.queryInputDir";
- public static final String PIR_STOPLIST_FILE = "test.pir.stopListFile";
- public static final String ES_PIR_INPUT_RESOURCE_PROPERTY = "test.pir.es.resource";
-
- // Elastic Search
- public static final String ES_INPUT_NODES_PROPERTY = "es.nodes";
- public static final String ES_INPUT_PORT_PROPERTY = "es.port";
- public static final String ES_INPUT_INDEX_PROPERTY = "test.es.index";
- public static final String ES_INPUT_TYPE_PROPERTY = "test.es.type";
- public static final String ES_INPUT_RESOURCE_PROPERTY = "test.es.resource";
-
- // Output
- public static final String OUTPUT_DIRECTORY_PROPERTY = "test.outputHDFSFile";
-
- public static void main(String[] args) throws Exception
- {
- // create a cli object to handle all program inputs
- DistributedTestCLI cli = new DistributedTestCLI(args);
-
- logger.info("DistributedTest Suite Beginning");
- FileSystem fs = FileSystem.get(new Configuration());
-
- String jarFile = cli.getOptionValue("j");
- logger.info("jarFile = " + jarFile);
- SystemConfiguration.setProperty("jarFile", jarFile);
-
- List<JSONObject> dataElements = initialize(fs);
-
- // Pull off the properties and reset upon completion
- String dataSchemasProp = SystemConfiguration.getProperty("data.schemas", "none");
- String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
- String stopListFileProp = SystemConfiguration.getProperty("pir.stopListFile");
-
- test(fs, cli, dataElements);
-
- cleanup(fs, dataSchemasProp, querySchemasProp, stopListFileProp);
- logger.info("Distributed Test Suite Complete");
- }
-
- /**
- * Create all inputs
- */
- public static List<JSONObject> initialize(FileSystem fs) throws Exception
- {
- List<JSONObject> dataElements = Inputs.createPIRJSONInput(fs);
-
- String localStopListFile = Inputs.createPIRStopList(fs, true);
- SystemConfiguration.setProperty("pir.stopListFile", localStopListFile);
-
- Inputs.createSchemaFiles(fs, true, StopListFilter.class.getName());
-
- return dataElements;
- }
-
- /**
- * Execute Tests
- */
- public static void test(FileSystem fs, DistributedTestCLI cli, List<JSONObject> pirDataElements) throws Exception
- {
- if (cli.run("1:J"))
- {
- DistTestSuite.testJSONInputMR(fs, pirDataElements);
- }
- if (cli.run("1:E") || cli.run("1:ES"))
- {
- Inputs.createPIRESInput();
- if (cli.run("1:E"))
- {
- DistTestSuite.testESInputMR(fs, pirDataElements);
- }
- if (cli.run("1:ES"))
- {
- DistTestSuite.testESInputSpark(fs, pirDataElements);
- }
- }
- if (cli.run("1:JS"))
- {
- DistTestSuite.testJSONInputSpark(fs, pirDataElements);
- }
- }
-
- /**
- * Delete all necessary inputs, clean up
- */
- public static void cleanup(FileSystem fs, String dataSchemasProp, String querySchemasProp, String stopListProp) throws Exception
- {
- Inputs.deleteESInput();
- fs.close();
-
- SystemConfiguration.setProperty("pir.stopListFile", stopListProp);
-
- // Force the query and data schemas to load their original values
- if (!dataSchemasProp.equals("none"))
- {
- DataSchemaLoader.initialize();
- }
-
- if (!querySchemasProp.equals("none"))
- {
- QuerySchemaLoader.initialize();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java b/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java
deleted file mode 100644
index 58f835c..0000000
--- a/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java
+++ /dev/null
@@ -1,458 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.test.distributed.testsuite;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.pirk.encryption.Paillier;
-import org.apache.pirk.inputformat.hadoop.InputFormatConst;
-import org.apache.pirk.inputformat.hadoop.json.JSONInputFormatBase;
-import org.apache.pirk.querier.wideskies.Querier;
-import org.apache.pirk.querier.wideskies.decrypt.DecryptResponse;
-import org.apache.pirk.querier.wideskies.encrypt.EncryptQuery;
-import org.apache.pirk.query.wideskies.Query;
-import org.apache.pirk.query.wideskies.QueryInfo;
-import org.apache.pirk.responder.wideskies.ResponderProps;
-import org.apache.pirk.responder.wideskies.mapreduce.ComputeResponseTool;
-import org.apache.pirk.response.wideskies.Response;
-import org.apache.pirk.schema.response.QueryResponseJSON;
-import org.apache.pirk.serialization.HadoopFileSystemStore;
-import org.apache.pirk.test.distributed.DistributedTestDriver;
-import org.apache.pirk.test.utils.BaseTests;
-import org.apache.pirk.test.utils.Inputs;
-import org.apache.pirk.test.utils.TestUtils;
-import org.apache.pirk.utils.SystemConfiguration;
-import org.apache.spark.launcher.SparkLauncher;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Distributed test class for PIR
- *
- */
-public class DistTestSuite
-{
- private static final Logger logger = LoggerFactory.getLogger(DistTestSuite.class);
-
- // This method also tests all non-query specific configuration options/properties
- // for the MapReduce version of PIR
- public static void testJSONInputMR(FileSystem fs, List<JSONObject> dataElements) throws Exception
- {
- logger.info("Starting testJSONInputMR");
-
- // Pull original data and query schema properties
- SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
- SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
-
- SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
- SystemConfiguration.setProperty("pir.maxHitsPerSelector", "100");
-
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
-
- // Set up base configs
- SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.BASE_FORMAT);
- SystemConfiguration.setProperty("pir.inputData", SystemConfiguration.getProperty(DistributedTestDriver.JSON_PIR_INPUT_FILE_PROPERTY));
- SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:0");
-
- // Run tests
- SystemConfiguration.setProperty("pirTest.embedSelector", "true");
- BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
- BaseTests.testDNSIPQuery(dataElements, fs, false, true, 1);
-
- SystemConfiguration.setProperty("pirTest.embedSelector", "false");
- BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 2);
- BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
-
- BaseTests.testSRCIPQueryNoFilter(dataElements, fs, false, true, 2);
-
- // Test hit limits per selector
- SystemConfiguration.setProperty("pir.limitHitsPerSelector", "true");
- SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1");
- BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 3);
- SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
- SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
-
- // Test the local cache for modular exponentiation
- SystemConfiguration.setProperty("pir.useLocalCache", "true");
- BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
- BaseTests.testSRCIPQuery(dataElements, fs, false, true, 2);
- SystemConfiguration.setProperty("pir.useLocalCache", "false");
-
- // Change query for NXDOMAIN
- SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:3");
- BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 2);
- SystemConfiguration.setProperty("pirTest.embedSelector", "false");
- BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 2);
- SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:0");
-
- // Test the expTable cases
- SystemConfiguration.setProperty("pirTest.embedSelector", "true");
-
- // In memory table
- SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
- SystemConfiguration.setProperty("pirTest.useExpLookupTable", "true");
- BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
-
- // Create exp table in hdfs
- SystemConfiguration.setProperty("mapreduce.map.memory.mb", "10000");
- SystemConfiguration.setProperty("mapreduce.reduce.memory.mb", "10000");
- SystemConfiguration.setProperty("mapreduce.map.java.opts", "-Xmx9000m");
- SystemConfiguration.setProperty("mapreduce.reduce.java.opts", "-Xmx9000m");
-
- SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "true");
- SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
- SystemConfiguration.setProperty("pir.expCreationSplits", "50");
- SystemConfiguration.setProperty("pir.numExpLookupPartitions", "150");
- BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
-
- // Reset exp properties
- SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
- SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
-
- // Reset property
- SystemConfiguration.setProperty("pirTest.embedSelector", "true");
-
- // Test embedded QuerySchema
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
- BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
-
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
- BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
-
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
- BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
-
- logger.info("Completed testJSONInputMR");
- }
-
- public static void testESInputMR(FileSystem fs, List<JSONObject> dataElements) throws Exception
- {
- logger.info("Starting testESInputMR");
-
- SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
- SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
-
- SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
- SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
-
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
-
- // Set up ES configs
- SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.ES);
- SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:0");
- SystemConfiguration.setProperty("pir.esResource", SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_RESOURCE_PROPERTY));
-
- // Run tests
- SystemConfiguration.setProperty("pirTest.embedSelector", "true");
- BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1);
- BaseTests.testSRCIPQuery(dataElements, fs, false, true, 2);
- BaseTests.testDNSIPQuery(dataElements, fs, false, true, 1);
-
- SystemConfiguration.setProperty("pirTest.embedSelector", "false");
- BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 2);
- BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2);
-
- // Change query for NXDOMAIN
- SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:3");
-
- SystemConfiguration.setProperty("pirTest.embedSelector", "true");
- BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 3);
- SystemConfiguration.setProperty("pirTest.embedSelector", "false");
- BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 3);
-
- logger.info("Completed testESInputMR");
- }
-
- public static void testJSONInputSpark(FileSystem fs, List<JSONObject> dataElements) throws Exception
- {
- logger.info("Starting testJSONInputSpark");
-
- SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
- SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
- SystemConfiguration.setProperty("pir.useModExpJoin", "false");
-
- SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
- SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
-
- SystemConfiguration.setProperty("pir.numColMultPartitions", "20");
- SystemConfiguration.setProperty("pir.colMultReduceByKey", "false");
-
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
-
- // Set up JSON configs
- SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.BASE_FORMAT);
- SystemConfiguration.setProperty("pir.inputData", SystemConfiguration.getProperty(DistributedTestDriver.JSON_PIR_INPUT_FILE_PROPERTY));
- SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:0");
-
- // Run tests
- SystemConfiguration.setProperty("pirTest.embedSelector", "true");
- BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
- BaseTests.testDNSIPQuery(dataElements, fs, true, true, 1);
-
- SystemConfiguration.setProperty("pirTest.embedSelector", "false");
- BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 2);
- BaseTests.testDNSIPQuery(dataElements, fs, true, true, 2);
- BaseTests.testSRCIPQuery(dataElements, fs, true, true, 2);
-
- BaseTests.testSRCIPQueryNoFilter(dataElements, fs, true, true, 2);
-
- // Test embedded QuerySchema
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
- BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
-
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
- BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
-
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
- BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
- SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
-
- // Test pad columns
- SystemConfiguration.setProperty("pir.padEmptyColumns", "true");
- BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
- SystemConfiguration.setProperty("pir.padEmptyColumns", "false");
-
- // Test hit limits per selector
- SystemConfiguration.setProperty("pir.limitHitsPerSelector", "true");
- SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1");
- BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 3);
- SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
- SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
-
- // Test the local cache for modular exponentiation
- SystemConfiguration.setProperty("pirTest.embedSelector", "true");
- SystemConfiguration.setProperty("pir.useLocalCache", "true");
- BaseTests.testDNSIPQuery(dataElements, fs, true, true, 3);
-
- // Test the join functionality for the modular exponentiation table
- SystemConfiguration.setProperty("pir.useModExpJoin", "true");
- BaseTests.testDNSIPQuery(dataElements, fs, true, true, 3);
- SystemConfiguration.setProperty("pir.useModExpJoin", "false");
-
- // Test file based exp lookup table for modular exponentiation
- SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "true");
- SystemConfiguration.setProperty("pir.expCreationSplits", "500");
- SystemConfiguration.setProperty("pir.numExpLookupPartitions", "150");
- BaseTests.testDNSIPQuery(dataElements, fs, true, true, 2);
- SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
-
- // Change query for NXDOMAIN
- SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:3");
-
- SystemConfiguration.setProperty("pirTest.embedSelector", "true");
- BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
- SystemConfiguration.setProperty("pirTest.embedSelector", "false");
- BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
-
- // Test with reduceByKey for column mult
- SystemConfiguration.setProperty("pir.colMultReduceByKey", "true");
- BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
-
- logger.info("Completed testJSONInputSpark");
- }
-
- public static void testESInputSpark(FileSystem fs, List<JSONObject> dataElements) throws Exception
- {
- logger.info("Starting testESInputSpark");
-
- SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false");
- SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false");
-
- SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false");
- SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000");
-
- SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
- SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
-
- // Set up ES configs
- SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.ES);
- SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:0");
- SystemConfiguration.setProperty("pir.esResource", SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_RESOURCE_PROPERTY));
-
- // Run tests
- SystemConfiguration.setProperty("pirTest.embedSelector", "true");
- BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1);
- BaseTests.testDNSIPQuery(dataElements, fs, true, true, 1);
- BaseTests.testSRCIPQuery(dataElements, fs, true, true, 2);
-
- SystemConfiguration.setProperty("pirTest.embedSelector", "false");
- BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 2);
- BaseTests.testDNSIPQuery(dataElements, fs, true, true, 2);
-
- // Change query for NXDOMAIN
- SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:3");
-
- SystemConfiguration.setProperty("pirTest.embedSelector", "true");
- BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
- SystemConfiguration.setProperty("pirTest.embedSelector", "false");
- BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3);
-
- logger.info("Completed testESInputSpark");
- }
-
- // Base method to perform query
- public static List<QueryResponseJSON> performQuery(String queryType, ArrayList<String> selectors, FileSystem fs, boolean isSpark, int numThreads)
- throws Exception
- {
- logger.info("performQuery: ");
-
- String queryInputDir = SystemConfiguration.getProperty(DistributedTestDriver.PIR_QUERY_INPUT_DIR);
- String outputFile = SystemConfiguration.getProperty(DistributedTestDriver.OUTPUT_DIRECTORY_PROPERTY);
- fs.delete(new Path(outputFile), true); // Ensure old output does not exist.
-
- SystemConfiguration.setProperty("pir.queryInput", queryInputDir);
- SystemConfiguration.setProperty("pir.outputFile", outputFile);
- SystemConfiguration.setProperty("pir.numReduceTasks", "1");
- SystemConfiguration.setProperty("pir.stopListFile", SystemConfiguration.getProperty(DistributedTestDriver.PIR_STOPLIST_FILE));
-
- // Create the temp result file
- File fileFinalResults = File.createTempFile("finalResultsFile", ".txt");
- fileFinalResults.deleteOnExit();
- logger.info("fileFinalResults = " + fileFinalResults.getAbsolutePath());
-
- boolean embedSelector = SystemConfiguration.getBooleanProperty("pirTest.embedSelector", false);
- boolean useExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useExpLookupTable", false);
- boolean useHDFSExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useHDFSExpLookupTable", false);
-
- // Set the necessary objects
- QueryInfo queryInfo = new QueryInfo(BaseTests.queryIdentifier, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize,
- queryType, useExpLookupTable, embedSelector, useHDFSExpLookupTable);
-
- Paillier paillier = new Paillier(BaseTests.paillierBitSize, BaseTests.certainty);
-
- // Perform the encryption
- logger.info("Performing encryption of the selectors - forming encrypted query vectors:");
- EncryptQuery encryptQuery = new EncryptQuery(queryInfo, selectors, paillier);
- encryptQuery.encrypt(numThreads);
- logger.info("Completed encryption of the selectors - completed formation of the encrypted query vectors:");
-
- // Grab the necessary objects
- Querier querier = encryptQuery.getQuerier();
- Query query = encryptQuery.getQuery();
-
- // Write the Querier object to a file
- Path queryInputDirPath = new Path(queryInputDir);
- new HadoopFileSystemStore(fs).store(queryInputDirPath, query);
- fs.deleteOnExit(queryInputDirPath);
-
- // Grab the original data and query schema properties to reset upon completion
- String dataSchemaProp = SystemConfiguration.getProperty("data.schemas");
- String querySchemaProp = SystemConfiguration.getProperty("query.schemas");
-
- // Get the correct input format class name
- JSONInputFormatBase jFormat = new JSONInputFormatBase();
- String jsonBaseInputFormatString = jFormat.getClass().getName();
- SystemConfiguration.setProperty("pir.baseInputFormat", jsonBaseInputFormatString);
-
- // Submitting the tool for encrypted query
- logger.info("Performing encrypted query:");
- if (isSpark)
- {
- // Build args
- String inputFormat = SystemConfiguration.getProperty("pir.dataInputFormat");
- logger.info("inputFormat = " + inputFormat);
- ArrayList<String> args = new ArrayList<>();
- args.add("-" + ResponderProps.PLATFORM + "=spark");
- args.add("-" + ResponderProps.DATAINPUTFORMAT + "=" + inputFormat);
- args.add("-" + ResponderProps.QUERYINPUT + "=" + SystemConfiguration.getProperty("pir.queryInput"));
- args.add("-" + ResponderProps.OUTPUTFILE + "=" + SystemConfiguration.getProperty("pir.outputFile"));
- args.add("-" + ResponderProps.STOPLISTFILE + "=" + SystemConfiguration.getProperty("pir.stopListFile"));
- args.add("-" + ResponderProps.USELOCALCACHE + "=" + SystemConfiguration.getProperty("pir.useLocalCache", "true"));
- args.add("-" + ResponderProps.LIMITHITSPERSELECTOR + "=" + SystemConfiguration.getProperty("pir.limitHitsPerSelector", "false"));
- args.add("-" + ResponderProps.MAXHITSPERSELECTOR + "=" + SystemConfiguration.getProperty("pir.maxHitsPerSelector", "1000"));
- args.add("-" + ResponderProps.QUERYSCHEMAS + "=" + Inputs.HDFS_QUERY_FILES);
- args.add("-" + ResponderProps.DATASCHEMAS + "=" + Inputs.DATA_SCHEMA_FILE_HDFS);
- args.add("-" + ResponderProps.NUMEXPLOOKUPPARTS + "=" + SystemConfiguration.getProperty("pir.numExpLookupPartitions", "100"));
- args.add("-" + ResponderProps.USEMODEXPJOIN + "=" + SystemConfiguration.getProperty("pir.useModExpJoin", "false"));
- args.add("-" + ResponderProps.NUMCOLMULTPARTITIONS + "=" + SystemConfiguration.getProperty("pir.numColMultPartitions", "20"));
- args.add("-" + ResponderProps.COLMULTREDUCEBYKEY + "=" + SystemConfiguration.getProperty("pir.colMultReduceByKey", "false"));
- if (inputFormat.equals(InputFormatConst.BASE_FORMAT))
- {
- args.add("-" + ResponderProps.INPUTDATA + "=" + SystemConfiguration.getProperty("pir.inputData"));
- args.add("-" + ResponderProps.BASEQUERY + "=" + SystemConfiguration.getProperty("pir.baseQuery"));
- args.add("-" + ResponderProps.BASEINPUTFORMAT + "=" + SystemConfiguration.getProperty("pir.baseInputFormat"));
- }
- else if (inputFormat.equals(InputFormatConst.ES))
- {
- args.add("-" + ResponderProps.ESQUERY + "=" + SystemConfiguration.getProperty("pir.esQuery"));
- args.add("-" + ResponderProps.ESRESOURCE + "=" + SystemConfiguration.getProperty("pir.esResource"));
- }
-
- for (String arg : args)
- {
- logger.info("arg = " + arg);
- }
-
- // Run spark application
- Process sLauncher = new SparkLauncher().setAppResource(SystemConfiguration.getProperty("jarFile"))
- .setSparkHome(SystemConfiguration.getProperty("spark.home")).setMainClass("org.apache.pirk.responder.wideskies.ResponderDriver")
- .addAppArgs(args.toArray(new String[args.size()])).setMaster("yarn-cluster").setConf(SparkLauncher.EXECUTOR_MEMORY, "2g")
- .setConf(SparkLauncher.DRIVER_MEMORY, "2g").setConf(SparkLauncher.EXECUTOR_CORES, "1").launch();
- sLauncher.waitFor();
- }
- else
- {
- SystemConfiguration.setProperty("data.schemas", Inputs.DATA_SCHEMA_FILE_HDFS);
- SystemConfiguration.setProperty("query.schemas", Inputs.HDFS_QUERY_FILES);
-
- ComputeResponseTool responseTool = new ComputeResponseTool();
- ToolRunner.run(responseTool, new String[] {});
- }
- logger.info("Completed encrypted query");
-
- // Perform decryption
- // Reconstruct the necessary objects from the files
- logger.info("Performing decryption; writing final results file");
- Response response = new HadoopFileSystemStore(fs).recall(outputFile, Response.class);
-
- // Perform decryption and output the result file
- DecryptResponse decryptResponse = new DecryptResponse(response, querier);
- decryptResponse.decrypt(numThreads);
- decryptResponse.writeResultFile(fileFinalResults);
- logger.info("Completed performing decryption and writing final results file");
-
- // Read in results
- logger.info("Reading in and checking results");
- List<QueryResponseJSON> results = TestUtils.readResultsFile(fileFinalResults);
-
- // Reset data and query schema properties
- SystemConfiguration.setProperty("data.schemas", dataSchemaProp);
- SystemConfiguration.setProperty("query.schemas", querySchemaProp);
-
- // Clean up output dir in hdfs
- fs.delete(new Path(outputFile), true);
-
- return results;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/main/java/org/apache/pirk/test/utils/BaseTests.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/pirk/test/utils/BaseTests.java b/src/main/java/org/apache/pirk/test/utils/BaseTests.java
deleted file mode 100644
index a55ed4d..0000000
--- a/src/main/java/org/apache/pirk/test/utils/BaseTests.java
+++ /dev/null
@@ -1,643 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.test.utils;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.pirk.query.wideskies.QueryUtils;
-import org.apache.pirk.schema.query.QuerySchema;
-import org.apache.pirk.schema.query.QuerySchemaRegistry;
-import org.apache.pirk.schema.response.QueryResponseJSON;
-import org.apache.pirk.test.distributed.testsuite.DistTestSuite;
-import org.apache.pirk.utils.StringUtils;
-import org.apache.pirk.utils.SystemConfiguration;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-
-import static org.junit.Assert.fail;
-
-/**
- * Class to hold the base functional distributed tests
- */
-public class BaseTests
-{
- private static final Logger logger = LoggerFactory.getLogger(BaseTests.class);
-
- public static final UUID queryIdentifier = UUID.randomUUID();
- public static final int dataPartitionBitSize = 8;
-
- // Selectors for domain and IP queries, queryIdentifier is the first entry for file generation
- private static ArrayList<String> selectorsDomain = new ArrayList<>(Arrays.asList("s.t.u.net", "d.e.com", "r.r.r.r", "a.b.c.com", "something.else", "x.y.net"));
- private static ArrayList<String> selectorsIP = new ArrayList<>(Arrays.asList("55.55.55.55", "5.6.7.8", "10.20.30.40", "13.14.15.16", "21.22.23.24"));
-
- // Encryption variables -- Paillier mechanisms are tested in the Paillier test code, so these are fixed...
- public static final int hashBitSize = 12;
- public static final String hashKey = "someKey";
- public static final int paillierBitSize = 384;
- public static final int certainty = 128;
-
- public static void testDNSHostnameQuery(ArrayList<JSONObject> dataElements, int numThreads, boolean testFalsePositive) throws Exception
- {
- testDNSHostnameQuery(dataElements, null, false, false, numThreads, testFalsePositive);
- }
-
- public static void testDNSHostnameQuery(List<JSONObject> dataElements, FileSystem fs, boolean isSpark, boolean isDistributed, int numThreads)
- throws Exception
- {
- testDNSHostnameQuery(dataElements, fs, isSpark, isDistributed, numThreads, false);
- }
-
- // Query for the watched hostname occurred; ; watched value type: hostname (String)
- public static void testDNSHostnameQuery(List<JSONObject> dataElements, FileSystem fs, boolean isSpark, boolean isDistributed, int numThreads,
- boolean testFalsePositive) throws Exception
- {
- logger.info("Running testDNSHostnameQuery(): ");
-
- QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_HOSTNAME_QUERY);
-
- int numExpectedResults = 6;
- List<QueryResponseJSON> results;
- if (isDistributed)
- {
- results = DistTestSuite.performQuery(Inputs.DNS_HOSTNAME_QUERY, selectorsDomain, fs, isSpark, numThreads);
- }
- else
- {
- results = StandaloneQuery.performStandaloneQuery(dataElements, Inputs.DNS_HOSTNAME_QUERY, selectorsDomain, numThreads, testFalsePositive);
- if (!testFalsePositive)
- {
- numExpectedResults = 7; // all 7 for non distributed case; if testFalsePositive==true, then 6
- }
- }
- logger.info("results:");
- printResultList(results);
-
- if (isDistributed && SystemConfiguration.isSetTrue("pir.limitHitsPerSelector"))
- {
- // 3 elements returned - one for each qname -- a.b.c.com, d.e.com, something.else
- if (results.size() != 3)
- {
- fail("results.size() = " + results.size() + " -- must equal 3");
- }
-
- // Check that each qname appears once in the result set
- HashSet<String> correctQnames = new HashSet<>();
- correctQnames.add("a.b.c.com");
- correctQnames.add("d.e.com");
- correctQnames.add("something.else");
-
- HashSet<String> resultQnames = new HashSet<>();
- for (QueryResponseJSON qrJSON : results)
- {
- resultQnames.add((String) qrJSON.getValue(Inputs.QNAME));
- }
-
- if (correctQnames.size() != resultQnames.size())
- {
- fail("correctQnames.size() = " + correctQnames.size() + " != resultQnames.size() " + resultQnames.size());
- }
-
- for (String resultQname : resultQnames)
- {
- if (!correctQnames.contains(resultQname))
- {
- fail("correctQnames does not contain resultQname = " + resultQname);
- }
- }
- }
- else
- {
- if (results.size() != numExpectedResults)
- {
- fail("results.size() = " + results.size() + " -- must equal " + numExpectedResults);
- }
-
- // Number of original elements at the end of the list that we do not need to consider for hits
- int removeTailElements = 2; // the last two data elements should not hit
- if (testFalsePositive)
- {
- removeTailElements = 3;
- }
-
- ArrayList<QueryResponseJSON> correctResults = new ArrayList<>();
- int i = 0;
- while (i < (dataElements.size() - removeTailElements))
- {
- JSONObject dataMap = dataElements.get(i);
-
- boolean addElement = true;
- if (isDistributed && dataMap.get(Inputs.RCODE).toString().equals("3"))
- {
- addElement = false;
- }
- if (addElement)
- {
- QueryResponseJSON wlJSON = new QueryResponseJSON();
- wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier.toString());
- wlJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_HOSTNAME_QUERY);
- wlJSON.setMapping(Inputs.DATE, dataMap.get(Inputs.DATE));
- wlJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP));
- wlJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
- wlJSON.setMapping(Inputs.QNAME, dataMap.get(Inputs.QNAME)); // this gets re-embedded as the original selector after decryption
- wlJSON.setMapping(Inputs.QTYPE, parseShortArray(dataMap, Inputs.QTYPE));
- wlJSON.setMapping(Inputs.RCODE, dataMap.get(Inputs.RCODE));
- wlJSON.setMapping(Inputs.IPS, parseArray(dataMap, Inputs.IPS, true));
- wlJSON.setMapping(QueryResponseJSON.SELECTOR, QueryUtils.getSelectorByQueryTypeJSON(qSchema, dataMap));
- correctResults.add(wlJSON);
- }
- ++i;
- }
- logger.info("correctResults: ");
- printResultList(correctResults);
-
- if (results.size() != correctResults.size())
- {
- logger.info("correctResults:");
- printResultList(correctResults);
- fail("results.size() = " + results.size() + " != correctResults.size() = " + correctResults.size());
- }
- for (QueryResponseJSON result : results)
- {
- if (!compareResultArray(correctResults, result))
- {
- fail("correctResults does not contain result = " + result.toString());
- }
- }
- }
- logger.info("Completed testDNSHostnameQuery(): ");
- }
-
- public static void testDNSIPQuery(ArrayList<JSONObject> dataElements, int numThreads) throws Exception
- {
- testDNSIPQuery(dataElements, null, false, false, numThreads);
- }
-
- // The watched IP address was detected in the response to a query; watched value type: IP address (String)
- public static void testDNSIPQuery(List<JSONObject> dataElements, FileSystem fs, boolean isSpark, boolean isDistributed, int numThreads) throws Exception
- {
- logger.info("Running testDNSIPQuery(): ");
-
- QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_IP_QUERY);
- List<QueryResponseJSON> results;
-
- if (isDistributed)
- {
- results = DistTestSuite.performQuery(Inputs.DNS_IP_QUERY, selectorsIP, fs, isSpark, numThreads);
-
- if (results.size() != 5)
- {
- fail("results.size() = " + results.size() + " -- must equal 5");
- }
- }
- else
- {
- results = StandaloneQuery.performStandaloneQuery(dataElements, Inputs.DNS_IP_QUERY, selectorsIP, numThreads, false);
-
- if (results.size() != 6)
- {
- fail("results.size() = " + results.size() + " -- must equal 6");
- }
- }
- printResultList(results);
-
- ArrayList<QueryResponseJSON> correctResults = new ArrayList<>();
- int i = 0;
- while (i < (dataElements.size() - 3)) // last three data elements not hit - one on stoplist, two don't match selectors
- {
- JSONObject dataMap = dataElements.get(i);
-
- boolean addElement = true;
- if (isDistributed && dataMap.get(Inputs.RCODE).toString().equals("3"))
- {
- addElement = false;
- }
- if (addElement)
- {
- QueryResponseJSON wlJSON = new QueryResponseJSON();
- wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier);
- wlJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_IP_QUERY);
- wlJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP));
- wlJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
- wlJSON.setMapping(Inputs.IPS, parseArray(dataMap, Inputs.IPS, true));
- wlJSON.setMapping(QueryResponseJSON.SELECTOR, QueryUtils.getSelectorByQueryTypeJSON(qSchema, dataMap));
- correctResults.add(wlJSON);
- }
- ++i;
- }
- if (results.size() != correctResults.size())
- {
- logger.info("correctResults:");
- printResultList(correctResults);
- fail("results.size() = " + results.size() + " != correctResults.size() = " + correctResults.size());
- }
- for (QueryResponseJSON result : results)
- {
- if (!compareResultArray(correctResults, result))
- {
- fail("correctResults does not contain result = " + result.toString());
- }
- }
- logger.info("Completed testDNSIPQuery(): ");
- }
-
- public static void testDNSNXDOMAINQuery(ArrayList<JSONObject> dataElements, int numThreads) throws Exception
- {
- testDNSNXDOMAINQuery(dataElements, null, false, false, numThreads);
- }
-
- // A query that returned an nxdomain response was made for the watched hostname; watched value type: hostname (String)
- public static void testDNSNXDOMAINQuery(List<JSONObject> dataElements, FileSystem fs, boolean isSpark, boolean isDistributed, int numThreads)
- throws Exception
- {
- logger.info("Running testDNSNXDOMAINQuery(): ");
-
- QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_NXDOMAIN_QUERY);
- List<QueryResponseJSON> results;
-
- if (isDistributed)
- {
- results = DistTestSuite.performQuery(Inputs.DNS_NXDOMAIN_QUERY, selectorsDomain, fs, isSpark, numThreads);
- }
- else
- {
- results = StandaloneQuery.performStandaloneQuery(dataElements, Inputs.DNS_NXDOMAIN_QUERY, selectorsDomain, numThreads, false);
- }
- printResultList(results);
-
- if (results.size() != 1)
- {
- fail("results.size() = " + results.size() + " -- must equal 1");
- }
-
- ArrayList<QueryResponseJSON> correctResults = new ArrayList<>();
- int i = 0;
- while (i < dataElements.size())
- {
- JSONObject dataMap = dataElements.get(i);
-
- if (dataMap.get(Inputs.RCODE).toString().equals("3"))
- {
- QueryResponseJSON wlJSON = new QueryResponseJSON();
- wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier);
- wlJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_NXDOMAIN_QUERY);
- wlJSON.setMapping(Inputs.QNAME, dataMap.get(Inputs.QNAME)); // this gets re-embedded as the original selector after decryption
- wlJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
- wlJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP));
- wlJSON.setMapping(QueryResponseJSON.SELECTOR, QueryUtils.getSelectorByQueryTypeJSON(qSchema, dataMap));
- correctResults.add(wlJSON);
- }
- ++i;
- }
- if (results.size() != correctResults.size())
- {
- logger.info("correctResults:");
- printResultList(correctResults);
- fail("results.size() = " + results.size() + " != correctResults.size() = " + correctResults.size());
- }
- for (QueryResponseJSON result : results)
- {
- if (!compareResultArray(correctResults, result))
- {
- fail("correctResults does not contain result = " + result.toString());
- }
- }
- logger.info("Completed testDNSNXDOMAINQuery(): ");
- }
-
- public static void testSRCIPQuery(ArrayList<JSONObject> dataElements, int numThreads) throws Exception
- {
- testSRCIPQuery(dataElements, null, false, false, numThreads);
- }
-
- // Query for responses from watched srcIPs
- public static void testSRCIPQuery(List<JSONObject> dataElements, FileSystem fs, boolean isSpark, boolean isDistributed, int numThreads) throws Exception
- {
- logger.info("Running testSRCIPQuery(): ");
-
- QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_SRCIP_QUERY);
- List<QueryResponseJSON> results;
-
- int removeTailElements = 0;
- int numExpectedResults = 1;
- if (isDistributed)
- {
- results = DistTestSuite.performQuery(Inputs.DNS_SRCIP_QUERY, selectorsIP, fs, isSpark, numThreads);
- removeTailElements = 2; // The last two elements are on the distributed stoplist
- }
- else
- {
- numExpectedResults = 3;
- results = StandaloneQuery.performStandaloneQuery(dataElements, Inputs.DNS_SRCIP_QUERY, selectorsIP, numThreads, false);
- }
- printResultList(results);
-
- if (results.size() != numExpectedResults)
- {
- fail("results.size() = " + results.size() + " -- must equal " + numExpectedResults);
- }
-
- ArrayList<QueryResponseJSON> correctResults = new ArrayList<>();
- int i = 0;
- while (i < (dataElements.size() - removeTailElements))
- {
- JSONObject dataMap = dataElements.get(i);
-
- boolean addElement = false;
- if (dataMap.get(Inputs.SRCIP).toString().equals("55.55.55.55") || dataMap.get(Inputs.SRCIP).toString().equals("5.6.7.8"))
- {
- addElement = true;
- }
- if (addElement)
- {
- // Form the correct result QueryResponseJSON object
- QueryResponseJSON qrJSON = new QueryResponseJSON();
- qrJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier);
- qrJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_SRCIP_QUERY);
- qrJSON.setMapping(Inputs.QNAME, parseString(dataMap, Inputs.QNAME));
- qrJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
- qrJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP));
- qrJSON.setMapping(Inputs.IPS, parseArray(dataMap, Inputs.IPS, true));
- qrJSON.setMapping(QueryResponseJSON.SELECTOR, QueryUtils.getSelectorByQueryTypeJSON(qSchema, dataMap));
- correctResults.add(qrJSON);
- }
- ++i;
- }
- logger.info("correctResults:");
- printResultList(correctResults);
-
- if (results.size() != correctResults.size())
- {
- logger.info("correctResults:");
- printResultList(correctResults);
- fail("results.size() = " + results.size() + " != correctResults.size() = " + correctResults.size());
- }
- for (QueryResponseJSON result : results)
- {
- if (!compareResultArray(correctResults, result))
- {
- fail("correctResults does not contain result = " + result.toString());
- }
- }
- logger.info("Completed testSRCIPQuery(): ");
- }
-
- // Query for responses from watched srcIPs
- public static void testSRCIPQueryNoFilter(List<JSONObject> dataElements, FileSystem fs, boolean isSpark, boolean isDistributed, int numThreads)
- throws Exception
- {
- logger.info("Running testSRCIPQueryNoFilter(): ");
-
- QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_SRCIP_QUERY_NO_FILTER);
- List<QueryResponseJSON> results;
-
- int numExpectedResults = 3;
- if (isDistributed)
- {
- results = DistTestSuite.performQuery(Inputs.DNS_SRCIP_QUERY_NO_FILTER, selectorsIP, fs, isSpark, numThreads);
- }
- else
- {
- results = StandaloneQuery.performStandaloneQuery(dataElements, Inputs.DNS_SRCIP_QUERY_NO_FILTER, selectorsIP, numThreads, false);
- }
- printResultList(results);
-
- if (results.size() != numExpectedResults)
- {
- fail("results.size() = " + results.size() + " -- must equal " + numExpectedResults);
- }
-
- ArrayList<QueryResponseJSON> correctResults = new ArrayList<>();
- int i = 0;
- while (i < dataElements.size())
- {
- JSONObject dataMap = dataElements.get(i);
-
- boolean addElement = false;
- if (dataMap.get(Inputs.SRCIP).toString().equals("55.55.55.55") || dataMap.get(Inputs.SRCIP).toString().equals("5.6.7.8"))
- {
- addElement = true;
- }
- if (addElement)
- {
- // Form the correct result QueryResponseJSON object
- QueryResponseJSON qrJSON = new QueryResponseJSON();
- qrJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier);
- qrJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_SRCIP_QUERY_NO_FILTER);
- qrJSON.setMapping(Inputs.QNAME, parseString(dataMap, Inputs.QNAME));
- qrJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP));
- qrJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP));
- qrJSON.setMapping(Inputs.IPS, parseArray(dataMap, Inputs.IPS, true));
- qrJSON.setMapping(QueryResponseJSON.SELECTOR, QueryUtils.getSelectorByQueryTypeJSON(qSchema, dataMap));
- correctResults.add(qrJSON);
- }
- ++i;
- }
- logger.info("correctResults:");
- printResultList(correctResults);
-
- if (results.size() != correctResults.size())
- {
- logger.info("correctResults:");
- printResultList(correctResults);
- fail("results.size() = " + results.size() + " != correctResults.size() = " + correctResults.size());
- }
- for (QueryResponseJSON result : results)
- {
- if (!compareResultArray(correctResults, result))
- {
- fail("correctResults does not contain result = " + result.toString());
- }
- }
- logger.info("Completed testSRCIPQueryNoFilter(): ");
- }
-
- @SuppressWarnings("unchecked")
- // Method to convert a ArrayList<String> into the correct (padded) returned ArrayList
- private static ArrayList<String> parseArray(JSONObject dataMap, String fieldName, boolean isIP)
- {
- ArrayList<String> retArray = new ArrayList<>();
-
- ArrayList<String> values;
- if (dataMap.get(fieldName) instanceof ArrayList)
- {
- values = (ArrayList<String>) dataMap.get(fieldName);
- }
- else
- {
- values = StringUtils.jsonArrayStringToArrayList((String) dataMap.get(fieldName));
- }
-
- int numArrayElementsToReturn = SystemConfiguration.getIntProperty("pir.numReturnArrayElements", 1);
- for (int i = 0; i < numArrayElementsToReturn; ++i)
- {
- if (i < values.size())
- {
- retArray.add(values.get(i));
- }
- else if (isIP)
- {
- retArray.add("0.0.0.0");
- }
- else
- {
- retArray.add("0");
- }
- }
-
- return retArray;
- }
-
- // Method to convert a ArrayList<Short> into the correct (padded) returned ArrayList
- private static ArrayList<Short> parseShortArray(JSONObject dataMap, String fieldName)
- {
- ArrayList<Short> retArray = new ArrayList<>();
-
- ArrayList<Short> values = (ArrayList<Short>) dataMap.get(fieldName);
-
- int numArrayElementsToReturn = SystemConfiguration.getIntProperty("pir.numReturnArrayElements", 1);
- for (int i = 0; i < numArrayElementsToReturn; ++i)
- {
- if (i < values.size())
- {
- retArray.add(values.get(i));
- }
- else
- {
- retArray.add((short) 0);
- }
- }
-
- return retArray;
- }
-
- // Method to convert the String field value to the correct returned substring
- private static String parseString(JSONObject dataMap, String fieldName)
- {
- String ret;
-
- String element = (String) dataMap.get(fieldName);
- int numParts = Integer.parseInt(SystemConfiguration.getProperty("pir.stringBits")) / dataPartitionBitSize;
- int len = numParts;
- if (element.length() < numParts)
- {
- len = element.length();
- }
- ret = new String(element.getBytes(), 0, len);
-
- return ret;
- }
-
- // Method to determine whether or not the correctResults contains an object equivalent to
- // the given result
- private static boolean compareResultArray(ArrayList<QueryResponseJSON> correctResults, QueryResponseJSON result)
- {
- boolean equivalent = false;
-
- for (QueryResponseJSON correct : correctResults)
- {
- equivalent = compareResults(correct, result);
- if (equivalent)
- {
- break;
- }
- }
-
- return equivalent;
- }
-
- // Method to test the equivalence of two test results
- private static boolean compareResults(QueryResponseJSON r1, QueryResponseJSON r2)
- {
- boolean equivalent = true;
-
- JSONObject jsonR1 = r1.getJSONObject();
- JSONObject jsonR2 = r2.getJSONObject();
-
- Set<String> r1KeySet = jsonR1.keySet();
- Set<String> r2KeySet = jsonR2.keySet();
- if (!r1KeySet.equals(r2KeySet))
- {
- equivalent = false;
- }
- if (equivalent)
- {
- for (String key : r1KeySet)
- {
- if (key.equals(Inputs.QTYPE) || key.equals(Inputs.IPS)) // array types
- {
- HashSet<String> set1 = getSetFromList(jsonR1.get(key));
- HashSet<String> set2 = getSetFromList(jsonR2.get(key));
-
- if (!set1.equals(set2))
- {
- equivalent = false;
- }
- }
- else
- {
- if (!(jsonR1.get(key).toString()).equals(jsonR2.get(key).toString()))
- {
- equivalent = false;
- }
- }
- }
- }
- return equivalent;
- }
-
- // Method to pull the elements of a list (either an ArrayList or JSONArray) into a HashSet
- private static HashSet<String> getSetFromList(Object list)
- {
- HashSet<String> set = new HashSet<>();
-
- if (list instanceof ArrayList)
- {
- for (Object obj : (ArrayList) list)
- {
- set.add(obj.toString());
- }
- }
- else
- // JSONArray
- {
- for (Object obj : (JSONArray) list)
- {
- set.add(obj.toString());
- }
- }
-
- return set;
- }
-
- private static void printResultList(List<QueryResponseJSON> list)
- {
- for (QueryResponseJSON obj : list)
- {
- logger.info(obj.toString());
- }
- }
-}
[4/6] incubator-pirk git commit: Move all tests under src/test
directory, this closes apache/incubator-pirk#66
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/general/QueryParserUtilsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/general/QueryParserUtilsTest.java b/src/test/java/org/apache/pirk/general/QueryParserUtilsTest.java
deleted file mode 100644
index 9ac2522..0000000
--- a/src/test/java/org/apache/pirk/general/QueryParserUtilsTest.java
+++ /dev/null
@@ -1,421 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.general;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.util.ArrayList;
-import java.util.Map;
-
-import org.apache.hadoop.io.MapWritable;
-import org.apache.pirk.schema.data.DataSchema;
-import org.apache.pirk.schema.data.DataSchemaRegistry;
-import org.apache.pirk.schema.query.QuerySchemaRegistry;
-import org.apache.pirk.test.utils.Inputs;
-import org.apache.pirk.utils.QueryParserUtils;
-import org.apache.pirk.utils.StringUtils;
-import org.apache.pirk.utils.SystemConfiguration;
-import org.json.simple.JSONObject;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Class for testing the QueryParser methods
- */
-public class QueryParserUtilsTest
-{
- private static final Logger logger = LoggerFactory.getLogger(QueryParserUtilsTest.class);
-
- private static MapWritable doc = null; // MapWritable with arrays in json string representation
- private static MapWritable docWAW = null; // MapWritable with arrays as WritableArrayWritable objects
- private static Map<String,Object> docMap = null; // arrays as ArrayList<String>
-
- private static DataSchema dSchema = null;
-
- @BeforeClass
- public static void setup() throws Exception
- {
- ArrayList<JSONObject> dataElementsJSON = Inputs.createJSONDataElements();
-
- // Reset the schema properties and registries
- DataSchemaRegistry.clearRegistry();
- QuerySchemaRegistry.clearRegistry();
- SystemConfiguration.setProperty("data.schemas", "none");
- SystemConfiguration.setProperty("query.schemas", "none");
-
- Inputs.createSchemaFiles(null, false, null);
-
- dSchema = DataSchemaRegistry.get(Inputs.TEST_DATA_SCHEMA_NAME);
-
- // ProcessBuilder pAdd1 = new ProcessBuilder("curl", "-XPUT", indexTypeNum1, "-d",
- // "{\"qname\":\"a.b.c.com\",\"date\":\"2016-02-20T23:29:05.000Z\",\"qtype\":[\"1\"]"
- // + ",\"rcode\":\"0\",\"src_ip\":\"55.55.55.55\",\"dest_ip\":\"1.2.3.6\"" + ",\"ip\":[\"10.20.30.40\",\"10.20.30.60\"]}");
- //
- doc = StringUtils.jsonStringToMapWritableWithArrayWritable(dataElementsJSON.get(0).toJSONString(), dSchema);
- docWAW = StringUtils.jsonStringToMapWritableWithWritableArrayWritable(dataElementsJSON.get(0).toJSONString(), dSchema);
- docMap = StringUtils.jsonStringToMap(dataElementsJSON.get(0).toJSONString(), dSchema);
- }
-
- @AfterClass
- public static void teardown()
- {
- // Reset the schema properties and registries
- DataSchemaRegistry.clearRegistry();
- QuerySchemaRegistry.clearRegistry();
- SystemConfiguration.setProperty("data.schemas", "none");
- SystemConfiguration.setProperty("query.schemas", "none");
- }
-
- @Test
- public void testSingleQuery()
- {
- String query1 = "?q=src_ip:55.55.55.55";
- assertTrue(QueryParserUtils.checkRecord(query1, doc, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query1, docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecord(query1, docMap, dSchema));
-
- String query2 = "?q=qname:a.b.c.com";
- assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema));
-
- String query3 = "?q=qname:d.b.c.com";
- assertFalse(QueryParserUtils.checkRecord(query3, doc, dSchema));
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema));
- assertFalse(QueryParserUtils.checkRecord(query3, docMap, dSchema));
- }
-
- @Test
- public void testQueryFieldDoesNotExist()
- {
- logger.info("running testQueryFieldDoesNotExist");
-
- // Field does not exist, this should not be found
- String query = "?q=nonexistent-field:*check*";
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query, docWAW, dSchema));
- assertFalse(QueryParserUtils.checkRecord(query, doc, dSchema));
- assertFalse(QueryParserUtils.checkRecord(query, docMap, dSchema));
-
- // First field does not exist, but second should be found
- String query2 = "?q=nonexistent-field:*check*+OR+qname:*a.b.c.com*";
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema));
-
- // First field does not exist, second field does, but AND operator makes query false
- String query3 = "?q=nonexistent-field:*check*+AND+qname:*a.b.c.com*";
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema));
- assertFalse(QueryParserUtils.checkRecord(query3, doc, dSchema));
- assertFalse(QueryParserUtils.checkRecord(query3, docMap, dSchema));
-
- logger.info("completed testQueryFieldDoesNotExist");
- }
-
- @Test
- public void testIgnoreCase()
- {
- logger.info("running testIgnoreCase");
-
- // with case sensitivity, should NOT be found
- String query = "?q=qname:*A.b.c.com*";
- assertFalse(QueryParserUtils.checkRecord(query, doc, dSchema));
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query, docWAW, dSchema));
- assertFalse(QueryParserUtils.checkRecord(query, docMap, dSchema));
-
- // with case sensitivity, should be found
- String query2 = "?q=qname:*a.b.c.com*";
- assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema));
-
- // adds @ flag = case insensitivity, thus should be found
- String query3 = "?q=qname@:*A.b.c.com*";
- assertTrue(QueryParserUtils.checkRecord(query3, doc, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecord(query3, docMap, dSchema));
-
- logger.info("completed testIgnoreCase");
- }
-
- @Test
- public void testSingleValueRangeQuery()
- {
- testSingleValueRangeQueryMapWritable();
- testSingleValueRangeQueryMap();
- testSingleValueRangeQueryMapWritableWAW();
- }
-
- private void testSingleValueRangeQueryMapWritable()
- {
- assertTrue(QueryParserUtils.checkRecord("?q=rcode:[0+TO+2]", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=rcode:{-1+TO+2}", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=rcode:[-1+TO+0]", doc, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=rcode:{0+TO+3}", doc, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=rcode:[3+TO+10]", doc, dSchema));
- }
-
- private void testSingleValueRangeQueryMap()
- {
- assertTrue(QueryParserUtils.checkRecord("?q=rcode:[0+TO+2]", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=rcode:{-1+TO+2}", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=rcode:[-1+TO+0]", docMap, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=rcode:{0+TO+3}", docMap, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=rcode:[3+TO+10]", docMap, dSchema));
- }
-
- private void testSingleValueRangeQueryMapWritableWAW()
- {
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[0+TO+2]", docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:{-1+TO+2}", docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[-1+TO+0]", docWAW, dSchema));
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:{0+TO+3}", docWAW, dSchema));
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[3+TO+10]", docWAW, dSchema));
- }
-
- @Test
- public void testIPRangeQuery()
- {
- testIPRangeQueryMapWritable();
- testIPRangeQueryMap();
- testIPRangeQueryMapWritableWAW();
- }
-
- public void testIPRangeQueryMapWritable()
- {
- // src_ip: 55.55.55.55
- // ip: 10.20.30.40,10.20.30.60
- assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", doc, dSchema));
-
- assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", doc, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", doc, dSchema));
-
- assertTrue(QueryParserUtils.checkRecord("?q=ip:[10.20.30.50+TO+10.20.30.69]", doc, dSchema));
- }
-
- public void testIPRangeQueryMapWritableWAW()
- {
- // src_ip: 55.55.55.55
- // ip: 10.20.30.40,10.20.30.60
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", docWAW, dSchema));
-
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", docWAW, dSchema));
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", docWAW, dSchema));
-
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=ip:[10.20.30.50+TO+10.20.30.69]", docWAW, dSchema));
- }
-
- public void testIPRangeQueryMap()
- {
- // src_ip: 55.55.55.55
- // ip: 10.20.30.40,10.20.30.60
- assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", docMap, dSchema));
-
- assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", docMap, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", docMap, dSchema));
-
- assertTrue(QueryParserUtils.checkRecord("?q=ip:[10.20.30.50+TO+10.20.30.69]", docMap, dSchema));
- }
-
- @Test
- public void testDateRangeQuery()
- {
- testDateRangeQueryMapWritable();
- testDateRangeQueryMapWritableWAW();
- testDateRangeQueryMap();
- }
-
- private void testDateRangeQueryMapWritable()
- {
- // date: 2016-02-20T23:29:05.000Z
-
- assertTrue(QueryParserUtils.checkRecord("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", doc, dSchema));
-
- assertFalse(QueryParserUtils.checkRecord("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", doc, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", doc, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", doc, dSchema));
- }
-
- private void testDateRangeQueryMap()
- {
- // date: 2016-02-20T23:29:05.000Z
-
- assertTrue(QueryParserUtils.checkRecord("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", docMap, dSchema));
-
- assertFalse(QueryParserUtils.checkRecord("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docMap, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", docMap, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", docMap, dSchema));
- }
-
- private void testDateRangeQueryMapWritableWAW()
- {
- // date: 2016-02-20T23:29:05.000Z
-
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", docWAW, dSchema));
-
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docWAW, dSchema));
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", docWAW, dSchema));
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", docWAW, dSchema));
- }
-
- @Test
- public void testBooleanQuery()
- {
- testBooleanQueryMapWritable();
- testBooleanQueryMapMapWritableWAW();
- testBooleanQueryMap();
- }
-
- private void testBooleanQueryMapWritable()
- {
- assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
-
- assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
-
- assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
-
- assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+qtype:2+OR+rcode:0", doc, dSchema));
- }
-
- private void testBooleanQueryMap()
- {
- assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
-
- assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
-
- assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
-
- assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+qtype:2+OR+rcode:0", docMap, dSchema));
- }
-
- private void testBooleanQueryMapMapWritableWAW()
- {
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema));
-
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]",
- docWAW, dSchema));
-
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]",
- docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]",
- docWAW, dSchema));
-
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:5+OR+qtype:2+OR+rcode:0", docWAW, dSchema));
- }
-
- @Test
- public void testAllQuery()
- {
- assertTrue(QueryParserUtils.checkRecord("?q=*", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=*", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=*", docWAW, dSchema));
- }
-
- @Test
- public void testWildcardQuery()
- {
- testWildcardQueryMapWritable();
- testWildcardQueryMap();
- testWildcardQueryMapWritableWAW();
- }
-
- private void testWildcardQueryMapWritable()
- {
- assertTrue(QueryParserUtils.checkRecord("?q=qname:*.com", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c*m", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b*", doc, dSchema));
-
- assertFalse(QueryParserUtils.checkRecord("?q=qname:*.org", doc, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=qname:mrtf*", doc, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljnik*.uk", doc, dSchema));
-
- assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c?m", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.?.com", doc, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=qname:?.b.c.com", doc, dSchema));
-
- assertFalse(QueryParserUtils.checkRecord("?q=qname:medelj?ikafera.com", doc, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljntkafer?.com", doc, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=qname:?edeljnikrfera.com", doc, dSchema));
- }
-
- private void testWildcardQueryMap()
- {
- assertTrue(QueryParserUtils.checkRecord("?q=qname:*.com", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c*m", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b*", docMap, dSchema));
-
- assertFalse(QueryParserUtils.checkRecord("?q=qname:*.org", docMap, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=qname:mrtf*", docMap, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljnik*.uk", docMap, dSchema));
-
- assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c?m", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.?.com", docMap, dSchema));
- assertTrue(QueryParserUtils.checkRecord("?q=qname:?.b.c.com", docMap, dSchema));
-
- assertFalse(QueryParserUtils.checkRecord("?q=qname:medelj?ikafera.com", docMap, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljntkafer?.com", docMap, dSchema));
- assertFalse(QueryParserUtils.checkRecord("?q=qname:?edeljnikrfera.com", docMap, dSchema));
- }
-
- private void testWildcardQueryMapWritableWAW()
- {
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:*.com", docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.c.c*m", docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b*", docWAW, dSchema));
-
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:*.org", docWAW, dSchema));
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:mrtf*", docWAW, dSchema));
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:nedeljnik*.uk", docWAW, dSchema));
-
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.c.c?m", docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.?.com", docWAW, dSchema));
- assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:?.b.c.com", docWAW, dSchema));
-
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:medelj?ikafera.com", docWAW, dSchema));
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:nedeljntkafer?.com", docWAW, dSchema));
- assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:?edeljnikrfera.com", docWAW, dSchema));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/schema/data/LoadDataSchemaTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/schema/data/LoadDataSchemaTest.java b/src/test/java/org/apache/pirk/schema/data/LoadDataSchemaTest.java
deleted file mode 100644
index 3aa500b..0000000
--- a/src/test/java/org/apache/pirk/schema/data/LoadDataSchemaTest.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.schema.data;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-
-import org.apache.pirk.schema.data.partitioner.IPDataPartitioner;
-import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner;
-import org.apache.pirk.test.utils.TestUtils;
-import org.apache.pirk.utils.SystemConfiguration;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-
-/**
- * Test suite for LoadDataSchema and DataSchema
- */
-public class LoadDataSchemaTest
-{
- private static final Logger logger = LoggerFactory.getLogger(LoadDataSchemaTest.class);
-
- private String dataSchemaName = "fakeDataSchema";
-
- private String element1 = "elementName1";
- private String element2 = "elementName2";
- private String element3 = "elementName3";
-
- @Test
- public void testGeneralSchemaLoad() throws Exception
- {
- // Pull off the property and reset upon completion
- String schemasProp = SystemConfiguration.getProperty("data.schemas", "none");
-
- // Write the schema file
- try
- {
- createDataSchema("schemaFile");
- } catch (IOException e)
- {
- e.printStackTrace();
- fail(e.toString());
- }
-
- // Force the schema to load
- DataSchemaLoader.initialize();
-
- // Check the entries
- DataSchema dSchema = DataSchemaRegistry.get(dataSchemaName);
-
- assertEquals(dataSchemaName, dSchema.getSchemaName());
-
- assertEquals(3, dSchema.getElementNames().size());
-
- // TODO: check Hadoop text names
-
- assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element1));
- assertEquals(PrimitiveTypePartitioner.INT, dSchema.getElementType(element2));
- assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element3));
-
- assertEquals(PrimitiveTypePartitioner.class.getName(), dSchema.getPartitionerTypeName(element1));
- if (!(dSchema.getPartitionerForElement(element1) instanceof PrimitiveTypePartitioner))
- {
- fail("Partitioner instance for element1 must be PrimitiveTypePartitioner");
- }
- assertEquals(IPDataPartitioner.class.getName(), dSchema.getPartitionerTypeName(element3));
- if (!(dSchema.getPartitionerForElement(element3) instanceof IPDataPartitioner))
- {
- fail("Partitioner instance for element3 must be IPDataPartitioner");
- }
-
- assertEquals(2, dSchema.getArrayElements().size());
- assertTrue(dSchema.getArrayElements().contains(element2));
- assertTrue(dSchema.getArrayElements().contains(element3));
-
- assertEquals(1, dSchema.getNonArrayElements().size());
- assertTrue(dSchema.getNonArrayElements().contains(element1));
-
- // Reset original data.schemas property
- SystemConfiguration.setProperty("data.schemas", schemasProp);
-
- // Force the schema to load
- if (!schemasProp.equals("none"))
- {
- DataSchemaLoader.initialize();
- }
- }
-
- @Test
- public void testIncorrectJavaType() throws Exception
- {
- // Pull off the property and reset upon completion
- String schemasProp = SystemConfiguration.getProperty("data.schemas");
-
- // Write the schema file
- try
- {
- createDataSchemaIncorrectJavaType("wrongJavaType");
- } catch (IOException e)
- {
- e.printStackTrace();
- fail(e.toString());
- }
-
- try
- {
- // Force the schema to load
- DataSchemaLoader.initialize();
- fail("DataSchemaLoader did not throw exception for incorrect javaType");
- } catch (Exception ignore)
- {}
-
- // Reset original data.schemas property
- SystemConfiguration.setProperty("data.schemas", schemasProp);
-
- // Force the schema to load
- DataSchemaLoader.initialize();
- }
-
- @Test
- public void testUnknownPartitioner() throws Exception
- {
- // Pull off the property and reset upon completion
- String schemasProp = SystemConfiguration.getProperty("data.schemas");
-
- // Write the schema file
- try
- {
- createDataSchemaUnknownPartitioner("unknownPartitioner");
- } catch (IOException e)
- {
- e.printStackTrace();
- fail(e.toString());
- }
-
- try
- {
- // Force the schema to load
- DataSchemaLoader.initialize();
- fail("DataSchemaLoader did not throw exception for unknown partitioner");
- } catch (Exception ignore)
- {}
-
- // Reset original data.schemas property
- SystemConfiguration.setProperty("data.schemas", schemasProp);
-
- // Force the schema to load
- DataSchemaLoader.initialize();
- }
-
- // Create the file that contains an unknown partitioner
- private void createDataSchemaUnknownPartitioner(String schemaFile) throws IOException
- {
- // Create a temporary file for the test schema, set in the properties
- File file = File.createTempFile(schemaFile, ".xml");
- file.deleteOnExit();
- logger.info("file = " + file.toString());
- SystemConfiguration.setProperty("data.schemas", file.toString());
-
- // Write to the file
- try
- {
- DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
- DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
- Document doc = dBuilder.newDocument();
-
- // root element
- Element rootElement = doc.createElement("schema");
- doc.appendChild(rootElement);
-
- // Add the schemaName
- Element schemaNameElement = doc.createElement("schemaName");
- schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
- rootElement.appendChild(schemaNameElement);
-
- // Add the element - unknown partitioner
- TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.INT, "false", "fakePartitioner");
-
- // Write to a xml file
- TransformerFactory transformerFactory = TransformerFactory.newInstance();
- Transformer transformer = transformerFactory.newTransformer();
- DOMSource source = new DOMSource(doc);
- StreamResult result = new StreamResult(file);
- transformer.transform(source, result);
-
- // Output for testing
- StreamResult consoleResult = new StreamResult(System.out);
- transformer.transform(source, consoleResult);
- System.out.println();
- } catch (Exception e)
- {
- e.printStackTrace();
- }
- }
-
- // Create the test data schema file
- private void createDataSchema(String schemaFile) throws IOException
- {
- // Create a temporary file for the test schema, set in the properties
- File file = File.createTempFile(schemaFile, ".xml");
- file.deleteOnExit();
- logger.info("file = " + file.toString());
- SystemConfiguration.setProperty("data.schemas", file.toString());
-
- // Write to the file
- try
- {
- DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
- DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
- Document doc = dBuilder.newDocument();
-
- // root element
- Element rootElement = doc.createElement("schema");
- doc.appendChild(rootElement);
-
- // Add the schemaName
- Element schemaNameElement = doc.createElement("schemaName");
- schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
- rootElement.appendChild(schemaNameElement);
-
- // Add the elements
- // element1 -- single String
- // TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.STRING, "false", PrimitiveTypePartitioner.class.getName());
- TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.STRING, "false", null);
-
- // element2 - -- array of Integers
- TestUtils.addElement(doc, rootElement, element2, PrimitiveTypePartitioner.INT, "true", PrimitiveTypePartitioner.class.getName());
-
- // element3 -- array of IP addresses
- TestUtils.addElement(doc, rootElement, element3, PrimitiveTypePartitioner.STRING, "true", IPDataPartitioner.class.getName());
-
- // Write to a xml file
- TransformerFactory transformerFactory = TransformerFactory.newInstance();
- Transformer transformer = transformerFactory.newTransformer();
- DOMSource source = new DOMSource(doc);
- StreamResult result = new StreamResult(file);
- transformer.transform(source, result);
-
- // Output for testing
- StreamResult consoleResult = new StreamResult(System.out);
- transformer.transform(source, consoleResult);
- System.out.println();
-
- } catch (Exception e)
- {
- e.printStackTrace();
- }
- }
-
- // Create the test schema file
- private void createDataSchemaIncorrectJavaType(String schemaFile) throws IOException
- {
- // Create a temporary file for the test schema, set in the properties
- File file = File.createTempFile(schemaFile, ".xml");
- file.deleteOnExit();
- logger.info("file = " + file.toString());
- SystemConfiguration.setProperty("data.schemas", file.toString());
-
- // Write to the file
- try
- {
- DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
- DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
- Document doc = dBuilder.newDocument();
-
- // root element
- Element rootElement = doc.createElement("schema");
- doc.appendChild(rootElement);
-
- // Add the schemaName
- Element schemaNameElement = doc.createElement("schemaName");
- schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
- rootElement.appendChild(schemaNameElement);
-
- // Add the element - unknown Java type
- TestUtils.addElement(doc, rootElement, element1, "bogus", "false", PrimitiveTypePartitioner.class.getName());
-
- // Write to a xml file
- TransformerFactory transformerFactory = TransformerFactory.newInstance();
- Transformer transformer = transformerFactory.newTransformer();
- DOMSource source = new DOMSource(doc);
- StreamResult result = new StreamResult(file);
- transformer.transform(source, result);
-
- // Output for testing
- StreamResult consoleResult = new StreamResult(System.out);
- transformer.transform(source, consoleResult);
- System.out.println();
-
- } catch (Exception e)
- {
- e.printStackTrace();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/schema/query/LoadQuerySchemaTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/schema/query/LoadQuerySchemaTest.java b/src/test/java/org/apache/pirk/schema/query/LoadQuerySchemaTest.java
deleted file mode 100644
index 55cb0a9..0000000
--- a/src/test/java/org/apache/pirk/schema/query/LoadQuerySchemaTest.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.schema.query;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-
-import org.apache.pirk.schema.data.DataSchemaLoader;
-import org.apache.pirk.schema.data.partitioner.IPDataPartitioner;
-import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner;
-import org.apache.pirk.schema.query.filter.StopListFilter;
-import org.apache.pirk.test.utils.Inputs;
-import org.apache.pirk.test.utils.TestUtils;
-import org.apache.pirk.utils.PIRException;
-import org.apache.pirk.utils.SystemConfiguration;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-
-/**
- * Test suite for LoadQuerySchema and QuerySchema
- */
-public class LoadQuerySchemaTest
-{
- private static final Logger logger = LoggerFactory.getLogger(LoadQuerySchemaTest.class);
-
- private String querySchemaFile = "querySchemaFile";
- private String dataSchemaName = "fakeDataSchema";
- private String querySchemaName = "fakeQuerySchema";
-
- private String element1 = "elementName1";
- private String element2 = "elementName2";
- private String element3 = "elementName3";
- private String element4 = "elementName4";
-
- private List<String> queryElements = Arrays.asList(element1, element2, element3);
- private List<String> filterElements = Collections.singletonList(element2);
-
- @Test
- public void testGeneralSchemaLoad() throws Exception
- {
- logger.info("Starting testGeneralSchemaLoad: ");
-
- // Pull off the properties and reset upon completion
- String dataSchemasProp = SystemConfiguration.getProperty("data.schemas", "none");
- String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
- String stopListFileProp = SystemConfiguration.getProperty("pir.stopListFile");
-
- // Create the stoplist file
- createStopListFile();
-
- // Create the data schema used and force it to load
- try
- {
- createDataSchema("dataSchemaFile");
- } catch (Exception e)
- {
- e.printStackTrace();
- fail(e.toString());
- }
- DataSchemaLoader.initialize();
-
- // Create the query schema used and force it to load
- try
- {
- TestUtils.createQuerySchema(querySchemaFile, querySchemaName, dataSchemaName, element4, queryElements, filterElements, StopListFilter.class.getName());
-
- } catch (IOException e)
- {
- e.printStackTrace();
- fail(e.toString());
- }
- QuerySchemaLoader.initialize();
-
- // Check the entries
- QuerySchema qSchema = QuerySchemaRegistry.get(querySchemaName);
-
- assertEquals(querySchemaName, qSchema.getSchemaName());
- assertEquals(dataSchemaName, qSchema.getDataSchemaName());
- assertEquals(element4, qSchema.getSelectorName());
-
- assertEquals(StopListFilter.class.getName(), qSchema.getFilterTypeName());
- if (!(qSchema.getFilter() instanceof StopListFilter))
- {
- fail("Filter class instance must be StopListFilter");
- }
-
- assertEquals(3, qSchema.getElementNames().size());
- for (String item : qSchema.getElementNames())
- {
- if (!(item.equals(element1) || item.equals(element2) || item.equals(element3)))
- {
- fail("elementNames: item = " + item + " must equal one of: " + element1 + ", " + element2 + ", or " + element3);
- }
- }
- assertEquals(1, qSchema.getFilteredElementNames().size());
- for (String item : qSchema.getFilteredElementNames())
- {
- if (!item.equals(element2))
- {
- fail("filterElementNames: item = " + item + " must equal " + element2);
- }
- }
-
- // one string, array IPs, array integers
- int stringSize = Integer.parseInt(SystemConfiguration.getProperty("pir.stringBits"));
- int arrayMult = Integer.parseInt(SystemConfiguration.getProperty("pir.numReturnArrayElements"));
- int dataElementSize = stringSize + 32 * arrayMult + 32 * arrayMult;
- assertEquals(dataElementSize, qSchema.getDataElementSize());
-
- // Reset original query and data schema properties
- SystemConfiguration.setProperty("data.schemas", dataSchemasProp);
- SystemConfiguration.setProperty("query.schemas", querySchemasProp);
- SystemConfiguration.setProperty("pir.stopListFile", stopListFileProp);
-
- // Force the query and data schemas to load their original values
- if (!dataSchemasProp.equals("none"))
- {
- DataSchemaLoader.initialize();
- }
-
- if (!querySchemasProp.equals("none"))
- {
- QuerySchemaLoader.initialize();
- }
-
- logger.info("Finished testGeneralSchemaLoad: ");
- }
-
- @Test
- public void testUnknownFilterClass() throws Exception
- {
- // Pull off the properties and reset upon completion
- String dataSchemasProp = SystemConfiguration.getProperty("data.schemas", "none");
- String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
-
- // Create the data schema used and force it to load
- try
- {
- createDataSchema("dataSchemaFile");
- } catch (Exception e)
- {
- e.printStackTrace();
- fail(e.toString());
- }
- DataSchemaLoader.initialize();
-
- // Create the query schema used and force it to load
- try
- {
- TestUtils.createQuerySchema(querySchemaFile, querySchemaName, dataSchemaName, "nonExistentElement", queryElements, filterElements, "bogusFilterClass");
-
- } catch (IOException e)
- {
- e.printStackTrace();
- fail(e.toString());
- }
- try
- {
- QuerySchemaLoader.initialize();
- fail("QuerySchemaLoader did not throw exception for bogus filter class");
- } catch (Exception ignore)
- {}
-
- // Reset original query and data schema properties
- SystemConfiguration.setProperty("data.schemas", dataSchemasProp);
- SystemConfiguration.setProperty("query.schemas", querySchemasProp);
-
- // Force the query and data schemas to load their original values
- if (!dataSchemasProp.equals("none"))
- {
- DataSchemaLoader.initialize();
- }
-
- if (!querySchemasProp.equals("none"))
- {
- QuerySchemaLoader.initialize();
- }
-
- logger.info("Finished testFunkyFilterScenarios");
- }
-
- @Test
- public void testDataSchemaDoesNotExist() throws Exception
- {
- logger.info("Starting testDataSchemaDoesNotExist: ");
-
- // Pull off the properties and reset upon completion
- String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
-
- // Create the query schema used and force it to load
- try
- {
- TestUtils.createQuerySchema(querySchemaFile, querySchemaName, dataSchemaName, element4, queryElements, filterElements, null);
-
- } catch (IOException e)
- {
- e.printStackTrace();
- fail(e.toString());
- }
- try
- {
- QuerySchemaLoader.initialize();
- fail("QuerySchemaLoader did not throw exception for non-existent DataSchema");
- } catch (Exception ignore)
- {}
-
- // Reset original query properties and force to load
- SystemConfiguration.setProperty("query.schemas", querySchemasProp);
- if (!querySchemasProp.equals("none"))
- {
- QuerySchemaLoader.initialize();
- }
-
- logger.info("Finished testDataSchemaDoesNotExist ");
- }
-
- @Test
- public void testSelectorDoesNotExistInDataSchema() throws Exception
- {
- logger.info("Starting testSelectorDoesNotExistInDataSchema: ");
-
- // Pull off the properties and reset upon completion
- String dataSchemasProp = SystemConfiguration.getProperty("data.schemas", "none");
- String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
-
- // Create the data schema used and force it to load
- try
- {
- createDataSchema("dataSchemaFile");
- } catch (Exception e)
- {
- e.printStackTrace();
- fail(e.toString());
- }
- DataSchemaLoader.initialize();
-
- // Create the query schema used and force it to load
- try
- {
- TestUtils.createQuerySchema(querySchemaFile, querySchemaName, dataSchemaName, "nonExistentElement", queryElements, filterElements,
- StopListFilter.class.getName());
-
- } catch (IOException e)
- {
- e.printStackTrace();
- fail(e.toString());
- }
- try
- {
- QuerySchemaLoader.initialize();
- fail("QuerySchemaLoader did not throw exception for non-existent selectorName");
- } catch (Exception ignore)
- {}
-
- // Reset original query and data schema properties
- SystemConfiguration.setProperty("data.schemas", dataSchemasProp);
- SystemConfiguration.setProperty("query.schemas", querySchemasProp);
-
- // Force the query and data schemas to load their original values
- if (!dataSchemasProp.equals("none"))
- {
- DataSchemaLoader.initialize();
- }
-
- if (!querySchemasProp.equals("none"))
- {
- QuerySchemaLoader.initialize();
- }
-
- logger.info("Finished testSelectorDoesNotExistInDataSchema ");
- }
-
- // Create the stoplist file and alter the properties accordingly
- private void createStopListFile() throws IOException, PIRException
- {
- SystemConfiguration.setProperty("pir.stopListFile", "testStopListFile");
- String newSLFile = Inputs.createPIRStopList(null, false);
- SystemConfiguration.setProperty("pir.stopListFile", newSLFile);
- }
-
- // Create the test data schema file
- private void createDataSchema(String schemaFile) throws IOException
- {
- // Create a temporary file for the test schema, set in the properties
- File file = File.createTempFile(schemaFile, ".xml");
- file.deleteOnExit();
- logger.info("file = " + file.toString());
- SystemConfiguration.setProperty("data.schemas", file.toString());
-
- // Write to the file
- try
- {
- DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
- DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
- Document doc = dBuilder.newDocument();
-
- // root element
- Element rootElement = doc.createElement("schema");
- doc.appendChild(rootElement);
-
- // Add the schemaName
- Element schemaNameElement = doc.createElement("schemaName");
- schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
- rootElement.appendChild(schemaNameElement);
-
- // Add the elements
- // element1 -- single String
- TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.STRING, "false", PrimitiveTypePartitioner.class.getName());
-
- // element2 - -- array of Integers
- TestUtils.addElement(doc, rootElement, element2, PrimitiveTypePartitioner.INT, "true", PrimitiveTypePartitioner.class.getName());
-
- // element3 -- array of IP addresses
- TestUtils.addElement(doc, rootElement, element3, PrimitiveTypePartitioner.STRING, "true", IPDataPartitioner.class.getName());
-
- // element4 -- single byte type
- TestUtils.addElement(doc, rootElement, element4, PrimitiveTypePartitioner.BYTE, "false", PrimitiveTypePartitioner.class.getName());
-
- // Write to a xml file
- TransformerFactory transformerFactory = TransformerFactory.newInstance();
- Transformer transformer = transformerFactory.newTransformer();
- DOMSource source = new DOMSource(doc);
- StreamResult result = new StreamResult(file);
- transformer.transform(source, result);
-
- // Output for testing
- StreamResult consoleResult = new StreamResult(System.out);
- transformer.transform(source, consoleResult);
- System.out.println();
-
- } catch (Exception e)
- {
- e.printStackTrace();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/serialization/SerializationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/serialization/SerializationTest.java b/src/test/java/org/apache/pirk/serialization/SerializationTest.java
deleted file mode 100644
index 8689d43..0000000
--- a/src/test/java/org/apache/pirk/serialization/SerializationTest.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pirk.serialization;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.Serializable;
-import java.util.Objects;
-import java.util.Random;
-
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class SerializationTest
-{
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- private static JsonSerializer jsonSerializer;
- private static JavaSerializer javaSerializer;
-
- @BeforeClass
- public static void setUp() throws Exception
- {
- jsonSerializer = new JsonSerializer();
- javaSerializer = new JavaSerializer();
- }
-
- @Test
- public void testJsonSerDe() throws Exception
- {
- File tempFile = folder.newFile("test-json-serialize");
- FileOutputStream fos = new FileOutputStream(tempFile);
- DummyRecord dummyRecord = new DummyRecord();
-
- jsonSerializer.write(fos, dummyRecord);
-
- FileInputStream fis = new FileInputStream(tempFile);
- Object deserializedDummyObject = jsonSerializer.read(fis, DummyRecord.class);
- Assert.assertEquals(dummyRecord, deserializedDummyObject);
- }
-
- @Test
- public void testJavaSerDe() throws Exception
- {
- File tempFile = folder.newFile("test-java-serialize");
- FileOutputStream fos = new FileOutputStream(tempFile);
- DummyRecord dummyRecord = new DummyRecord();
-
- javaSerializer.write(fos, new DummyRecord());
-
- FileInputStream fis = new FileInputStream(tempFile);
- Object deserializedDummyObject = javaSerializer.read(fis, DummyRecord.class);
- Assert.assertTrue(deserializedDummyObject.equals(dummyRecord));
- }
-
- private static class DummyRecord implements Serializable, Storable
- {
- private int id;
- private String message;
- private long seed = 100L;
-
- DummyRecord()
- {
- this.id = (new Random(seed)).nextInt(5);
- this.message = "The next message id is " + id;
- }
-
- public int getId()
- {
- return id;
- }
-
- public void setId(int id)
- {
- this.id = id;
- }
-
- public String getMessage()
- {
- return message;
- }
-
- public void setMessage(String message)
- {
- this.message = message;
- }
-
- @Override
- public String toString()
- {
- return "DummyRecord{" + "id=" + id + ", message='" + message + '\'' + '}';
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
- DummyRecord that = (DummyRecord) o;
- return id == that.id && Objects.equals(message, that.message);
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(id, message);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/benchmark/BenchmarkDriver.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/benchmark/BenchmarkDriver.java b/src/test/java/org/apache/pirk/test/benchmark/BenchmarkDriver.java
new file mode 100644
index 0000000..e71c487
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/benchmark/BenchmarkDriver.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pirk.test.benchmark;
+
+import java.io.IOException;
+
+import org.openjdk.jmh.Main;
+import org.openjdk.jmh.runner.RunnerException;
+
+/**
+ * Driver for JMH benchmarking
+ */
+public class BenchmarkDriver
+{
+ public static void main(String[] args) throws RunnerException, IOException
+ {
+ Main.main(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/benchmark/PaillierBenchmark.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/benchmark/PaillierBenchmark.java b/src/test/java/org/apache/pirk/test/benchmark/PaillierBenchmark.java
new file mode 100644
index 0000000..15533b5
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/benchmark/PaillierBenchmark.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pirk.test.benchmark;
+
+import java.math.BigInteger;
+
+import org.apache.pirk.encryption.ModPowAbstraction;
+import org.apache.pirk.encryption.Paillier;
+import org.apache.pirk.utils.PIRException;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A JMH benchmark to evaluate Paillier performance both with and without using com.square.jnagmp.gmp to accelerate modPow
+ * <p>
+ * Guides to using JMH can be found at: http://tutorials.jenkov.com/java-performance/jmh.html and http://nitschinger.at/Using-JMH-for-Java-Microbenchmarking/
+ */
+
+public class PaillierBenchmark
+{
+ private static final int MODULUS_SIZE = 3074;
+ private static final Logger logger = LoggerFactory.getLogger(PaillierBenchmark.class);
+
+ @State(Scope.Benchmark)
+ public static class PaillierBenchmarkState
+ {
+ BigInteger r1 = null; // random number in (Z/NZ)*
+ BigInteger m1 = null; // message to encrypt
+
+ Paillier pallier = null;
+
+ /**
+ * This sets up the state for the two separate benchmarks
+ */
+ @Setup(org.openjdk.jmh.annotations.Level.Trial)
+ public void setUp()
+ {
+ int systemPrimeCertainty = SystemConfiguration.getIntProperty("pir.primeCertainty", 100);
+ try
+ {
+ pallier = new Paillier(MODULUS_SIZE, systemPrimeCertainty);
+
+ } catch (PIRException e)
+ {
+ System.out.printf("Couldn't build pallier object!%n");
+ }
+
+ r1 = BigInteger.valueOf(3);
+ m1 = BigInteger.valueOf(5);
+ }
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ public void testWithGMP(PaillierBenchmarkState allState)
+ {
+ SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
+ SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "false");
+ ModPowAbstraction.reloadConfiguration();
+
+ try
+ {
+ allState.pallier.encrypt(allState.m1, allState.r1);
+ } catch (PIRException e)
+ {
+ logger.info("Exception in testWithGMP!\n");
+ }
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ public void testWithGMPConstantTime(PaillierBenchmarkState allState)
+ {
+ SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
+ SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "true");
+ ModPowAbstraction.reloadConfiguration();
+
+ try
+ {
+ allState.pallier.encrypt(allState.m1, allState.r1);
+ } catch (PIRException e)
+ {
+ logger.info("Exception in testWithGMPConstantTime!\n");
+ }
+ }
+
+ @Benchmark
+ @BenchmarkMode(Mode.Throughput)
+ public void testWithoutGMP(PaillierBenchmarkState allState)
+ {
+ SystemConfiguration.setProperty("paillier.useGMPForModPow", "false");
+ ModPowAbstraction.reloadConfiguration();
+
+ try
+ {
+ allState.pallier.encrypt(allState.m1, allState.r1);
+ } catch (PIRException e)
+ {
+ logger.info("Exception in testWithoutGMP!\n");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/distributed/DistributedTestCLI.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/distributed/DistributedTestCLI.java b/src/test/java/org/apache/pirk/test/distributed/DistributedTestCLI.java
new file mode 100644
index 0000000..1535e1f
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/distributed/DistributedTestCLI.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.distributed;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A container for Apache's Command Line Interface that contains custom functionality for the MapReduce functional tests.
+ */
+public class DistributedTestCLI
+{
+ private static final Logger logger = LoggerFactory.getLogger(DistributedTestCLI.class);
+
+ private CommandLine commandLine = null;
+ private Options cliOptions = null;
+
+ /**
+ * Create and parse allowable options
+ *
+ * @param args
+ * - arguments fed into the main method
+ */
+ public DistributedTestCLI(String[] args)
+ {
+ // create the command line options
+ cliOptions = createOptions();
+
+ try
+ {
+ // parse the command line options
+ CommandLineParser parser = new GnuParser();
+ commandLine = parser.parse(cliOptions, args, true);
+
+ // if help option is selected, just print help text and exit
+ if (hasOption("h"))
+ {
+ printHelp();
+ System.exit(1);
+ }
+
+ // The full path of the jar file must be set
+ if (!hasOption("j"))
+ {
+ logger.info("The full path of the jar file must be set with -j");
+ System.exit(1);
+ }
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ /**
+ * Determine if an option was provided by the user via the CLI
+ *
+ * @param option
+ * - the option of interest
+ * @return true if option was provided, false otherwise
+ */
+ public boolean hasOption(String option)
+ {
+ return commandLine.hasOption(option);
+ }
+
+ /**
+ * Obtain the argument of the option provided by the user via the CLI
+ *
+ * @param option
+ * - the option of interest
+ * @return value of the argument of the option
+ */
+ public String getOptionValue(String option)
+ {
+ return commandLine.getOptionValue(option);
+ }
+
+ /**
+ * Determine if the argument was provided, which determines if a test should or should not be run
+ *
+ * @param allowed
+ * - argument string you are looking for
+ * @return true if argument was provided via the CLI, false otherwise
+ */
+ public boolean run(String allowed)
+ {
+ return run(allowed, "t");
+ }
+
+ /**
+ * Determine if the argument was provided for the selected option, which determines if a test should or should not be run
+ *
+ * @param allowed
+ * - argument string you are looking for
+ * @param option
+ * - the option of interest
+ * @return true if argument was provided via the CLI, false otherwise
+ */
+ public boolean run(String allowed, String option)
+ {
+ if (!hasOption(option))
+ {
+ return true;
+ }
+
+ String selection = getOptionValue(option);
+ String[] selectionList = selection.split(",");
+
+ for (String selectionItem : selectionList)
+ {
+ if (selectionItem.equals(allowed))
+ {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Create the options available for the DistributedTestDriver
+ *
+ * @return Apache's CLI Options object
+ */
+ private Options createOptions()
+ {
+ Options options = new Options();
+
+ // help
+ Option optionHelp = new Option("h", "help", false, "Print out the help documentation for this command line execution");
+ optionHelp.setRequired(false);
+ options.addOption(optionHelp);
+
+ // jar file
+ Option optionJar = new Option("j", "jar", true, "required -- Fully qualified jar file");
+ optionJar.setRequired(false);
+ options.addOption(optionJar);
+
+ // test selection
+ String tests = "testNum = 1: Wideskies Tests\n";
+ tests += "Subtests:\n";
+ tests += "E - Elasticsearch MapReduce\n";
+ tests += "J - JSON/HDFS MapReduce\n";
+ tests += "ES - Elasticsearch Spark \n";
+ tests += "JS - JSON/HDFS Spark \n";
+
+ Option optionTestSelection = new Option("t", "tests", true, "optional -- Select which tests to execute: \n" + tests);
+ optionTestSelection.setRequired(false);
+ optionTestSelection.setArgName("<testNum>:<subtestDesignator>");
+ optionTestSelection.setType(String.class);
+ options.addOption(optionTestSelection);
+
+ return options;
+ }
+
+ /**
+ * Prints out the help message
+ */
+ private void printHelp()
+ {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.setWidth(140);
+ formatter.printHelp("DistributedTestDriver", cliOptions);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/a643ae68/src/test/java/org/apache/pirk/test/distributed/DistributedTestDriver.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/test/distributed/DistributedTestDriver.java b/src/test/java/org/apache/pirk/test/distributed/DistributedTestDriver.java
new file mode 100755
index 0000000..ee37e63
--- /dev/null
+++ b/src/test/java/org/apache/pirk/test/distributed/DistributedTestDriver.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.test.distributed;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.pirk.schema.data.DataSchemaLoader;
+import org.apache.pirk.schema.query.QuerySchemaLoader;
+import org.apache.pirk.schema.query.filter.StopListFilter;
+import org.apache.pirk.test.distributed.testsuite.DistTestSuite;
+import org.apache.pirk.test.utils.Inputs;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Driver class to run the suite of functional tests for MR and Spark PIR
+ *
+ */
+public class DistributedTestDriver
+{
+ private static final Logger logger = LoggerFactory.getLogger(DistributedTestDriver.class);
+
+ // Input
+ public static final String JSON_PIR_INPUT_FILE_PROPERTY = "test.pir.inputJSONFile";
+ public static final String ES_PIR_INPUT_INDEX_PROPERTY = "test.pir.es.index";
+ public static final String PIR_QUERY_INPUT_DIR = "test.pir.queryInputDir";
+ public static final String PIR_STOPLIST_FILE = "test.pir.stopListFile";
+ public static final String ES_PIR_INPUT_RESOURCE_PROPERTY = "test.pir.es.resource";
+
+ // Elastic Search
+ public static final String ES_INPUT_NODES_PROPERTY = "es.nodes";
+ public static final String ES_INPUT_PORT_PROPERTY = "es.port";
+ public static final String ES_INPUT_INDEX_PROPERTY = "test.es.index";
+ public static final String ES_INPUT_TYPE_PROPERTY = "test.es.type";
+ public static final String ES_INPUT_RESOURCE_PROPERTY = "test.es.resource";
+
+ // Output
+ public static final String OUTPUT_DIRECTORY_PROPERTY = "test.outputHDFSFile";
+
+ public static void main(String[] args) throws Exception
+ {
+ // create a cli object to handle all program inputs
+ DistributedTestCLI cli = new DistributedTestCLI(args);
+
+ logger.info("DistributedTest Suite Beginning");
+ FileSystem fs = FileSystem.get(new Configuration());
+
+ String jarFile = cli.getOptionValue("j");
+ logger.info("jarFile = " + jarFile);
+ SystemConfiguration.setProperty("jarFile", jarFile);
+
+ List<JSONObject> dataElements = initialize(fs);
+
+ // Pull off the properties and reset upon completion
+ String dataSchemasProp = SystemConfiguration.getProperty("data.schemas", "none");
+ String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
+ String stopListFileProp = SystemConfiguration.getProperty("pir.stopListFile");
+
+ test(fs, cli, dataElements);
+
+ cleanup(fs, dataSchemasProp, querySchemasProp, stopListFileProp);
+ logger.info("Distributed Test Suite Complete");
+ }
+
+ /**
+ * Create all inputs
+ */
+ public static List<JSONObject> initialize(FileSystem fs) throws Exception
+ {
+ List<JSONObject> dataElements = Inputs.createPIRJSONInput(fs);
+
+ String localStopListFile = Inputs.createPIRStopList(fs, true);
+ SystemConfiguration.setProperty("pir.stopListFile", localStopListFile);
+
+ Inputs.createSchemaFiles(fs, true, StopListFilter.class.getName());
+
+ return dataElements;
+ }
+
+ /**
+ * Execute Tests
+ */
+ public static void test(FileSystem fs, DistributedTestCLI cli, List<JSONObject> pirDataElements) throws Exception
+ {
+ if (cli.run("1:J"))
+ {
+ DistTestSuite.testJSONInputMR(fs, pirDataElements);
+ }
+ if (cli.run("1:E") || cli.run("1:ES"))
+ {
+ Inputs.createPIRESInput();
+ if (cli.run("1:E"))
+ {
+ DistTestSuite.testESInputMR(fs, pirDataElements);
+ }
+ if (cli.run("1:ES"))
+ {
+ DistTestSuite.testESInputSpark(fs, pirDataElements);
+ }
+ }
+ if (cli.run("1:JS"))
+ {
+ DistTestSuite.testJSONInputSpark(fs, pirDataElements);
+ }
+ }
+
+ /**
+ * Delete all necessary inputs, clean up
+ */
+ public static void cleanup(FileSystem fs, String dataSchemasProp, String querySchemasProp, String stopListProp) throws Exception
+ {
+ Inputs.deleteESInput();
+ fs.close();
+
+ SystemConfiguration.setProperty("pir.stopListFile", stopListProp);
+
+ // Force the query and data schemas to load their original values
+ if (!dataSchemasProp.equals("none"))
+ {
+ DataSchemaLoader.initialize();
+ }
+
+ if (!querySchemasProp.equals("none"))
+ {
+ QuerySchemaLoader.initialize();
+ }
+ }
+}