You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pirk.apache.org by ea...@apache.org on 2016/07/31 23:06:28 UTC
[2/9] incubator-pirk git commit: [PIRK-37]: Rename package for test
classes to be org.apache.pirk.* -- closes apache/incubator-pirk#39
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/0912bb65/src/test/java/org/apache/pirk/general/KeyedHashTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/general/KeyedHashTest.java b/src/test/java/org/apache/pirk/general/KeyedHashTest.java
new file mode 100644
index 0000000..676609f
--- /dev/null
+++ b/src/test/java/org/apache/pirk/general/KeyedHashTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.general;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.pirk.utils.KeyedHash;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Basic functional tests for KeyedHash
+ *
+ */
+public class KeyedHashTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(KeyedHashTest.class);
+
+ @Test
+ public void testKeyedHash()
+ {
+ logger.info("Starting testKeyedHash: ");
+
+ int hash1 = KeyedHash.hash("someKey", 12, "someInput");
+ logger.info("hash1 = " + hash1 + " hash1 = " + Integer.toString(hash1, 2));
+
+ int hash2 = KeyedHash.hash("someKey", 32, "someInput");
+ logger.info("hash2 = " + hash2 + " hash2 = " + Integer.toString(hash2, 2));
+
+ int hash3 = KeyedHash.hash("someKey", 34, "someInput");
+ logger.info("hash3 = " + hash3 + " hash3 = " + Integer.toString(hash3, 2));
+
+ assertEquals(hash2, hash3);
+ assertEquals(hash1, hash2 & 0xFFF);
+
+ logger.info("Successfully completed testKeyedHash");
+ }
+
+ @Test
+ public void testKeyedHashWithType()
+ {
+ testKeyedHashType("MD5");
+ testKeyedHashType("SHA-1");
+ testKeyedHashType("SHA-256");
+ testKeyedHashType("FAKE-HASH-TYPE");
+ }
+
+ private void testKeyedHashType(String type)
+ {
+ logger.info("Starting testKeyedHashType with type: " + type);
+
+ int hash1 = KeyedHash.hash("someKey", 12, "someInput", type);
+ logger.info("hash1 = " + hash1 + " hash1 = " + Integer.toString(hash1, 2));
+
+ int hash2 = KeyedHash.hash("someKey", 32, "someInput", type);
+ logger.info("hash2 = " + hash2 + " hash2 = " + Integer.toString(hash2, 2));
+
+ int hash3 = KeyedHash.hash("someKey", 34, "someInput", type);
+ logger.info("hash3 = " + hash3 + " hash3 = " + Integer.toString(hash3, 2));
+
+ assertEquals(hash2, hash3);
+ assertEquals(hash1, hash2 & 0xFFF);
+
+ logger.info("Successfully completed testKeyedHashType with type: " + type);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/0912bb65/src/test/java/org/apache/pirk/general/PaillierTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/general/PaillierTest.java b/src/test/java/org/apache/pirk/general/PaillierTest.java
new file mode 100644
index 0000000..5d1c6b2
--- /dev/null
+++ b/src/test/java/org/apache/pirk/general/PaillierTest.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.general;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.math.BigInteger;
+import java.util.Random;
+
+import org.apache.pirk.encryption.Paillier;
+import org.apache.pirk.utils.PIRException;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Basic test functionality for Paillier library
+ *
+ */
+public class PaillierTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(PaillierTest.class);
+
+ private static BigInteger p = null; // large prime
+ private static BigInteger q = null; // large prime
+ private static BigInteger N = null; // N=pq, RSA modulus
+ private static BigInteger NSquared = null; // N^2
+ private static BigInteger lambdaN = null; // lambda(N) = lcm(p-1,q-1)
+
+ private static int bitLength = 0; // bit length of the modulus N
+ private static int certainty = 64; // prob that new BigInteger values represents primes will exceed (1 - (1/2)^certainty)
+
+ private static BigInteger r1 = null; // random number in (Z/NZ)*
+ private static BigInteger r2 = null; // random number in (Z/NZ)*
+
+ private static BigInteger m1 = null; // message to encrypt
+ private static BigInteger m2 = null; // message to encrypt
+
+ @BeforeClass
+ public static void setup()
+ {
+ p = BigInteger.valueOf(7);
+ q = BigInteger.valueOf(17);
+ N = p.multiply(q);
+ NSquared = N.multiply(N);
+
+ lambdaN = BigInteger.valueOf(48);
+
+ r1 = BigInteger.valueOf(3);
+ r2 = BigInteger.valueOf(4);
+
+ m1 = BigInteger.valueOf(5);
+ m2 = BigInteger.valueOf(2);
+
+ bitLength = 201;// bitLength = 384;
+ certainty = 128;
+
+ logger.info("p = " + p.intValue() + " q = " + q.intValue() + " N = " + N.intValue() + " bitLength = " + N.bitLength() + " lambdaN = " + lambdaN + " m1 = "
+ + m1.intValue() + " m2 = " + m2.intValue() + " r1 = " + r1.intValue() + " r2 = " + r2.intValue());
+ }
+
+ @Test
+ @SuppressWarnings("unused")
+ public void testPIRExceptions()
+ {
+ try
+ {
+ Paillier paillier = new Paillier(BigInteger.valueOf(2), BigInteger.valueOf(2), 128);
+ fail("Paillier constructor did not throw PIRException for p,q < 3");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier paillier = new Paillier(BigInteger.valueOf(2), BigInteger.valueOf(3), 128);
+ fail("Paillier constructor did not throw PIRException for p < 3");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier paillier = new Paillier(BigInteger.valueOf(3), BigInteger.valueOf(2), 128);
+ fail("Paillier constructor did not throw PIRException for q < 3");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier paillier = new Paillier(BigInteger.valueOf(7), BigInteger.valueOf(7), 128);
+ fail("Paillier constructor did not throw PIRException for p = q");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier paillier = new Paillier(BigInteger.valueOf(8), BigInteger.valueOf(7), 128);
+ fail("Paillier constructor did not throw PIRException for p not prime");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier paillier = new Paillier(BigInteger.valueOf(7), BigInteger.valueOf(10), 128);
+ fail("Paillier constructor did not throw PIRException for q not prime");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ int systemPrimeCertainty = Integer.parseInt(SystemConfiguration.getProperty("pir.primeCertainty", "128"));
+ Paillier paillier = new Paillier(3072, systemPrimeCertainty - 10);
+ fail("Paillier constructor did not throw PIRException for certainty less than system default of " + systemPrimeCertainty);
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier pailler = new Paillier(p, q, bitLength);
+ BigInteger encM1 = pailler.encrypt(N);
+ fail("Paillier encryption did not throw PIRException for message m = N");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier pailler = new Paillier(p, q, bitLength);
+ BigInteger encM1 = pailler.encrypt(N.add(BigInteger.TEN));
+ fail("Paillier encryption did not throw PIRException for message m > N");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier pailler = new Paillier(bitLength, 128, bitLength);
+ fail("Paillier constructor did not throw PIRException for ensureBitSet = bitLength");
+ } catch (PIRException ignore)
+ {}
+
+ try
+ {
+ Paillier pailler = new Paillier(bitLength, 128, bitLength + 1);
+ fail("Paillier constructor did not throw PIRException for ensureBitSet > bitLength");
+ } catch (PIRException ignore)
+ {}
+ }
+
+ @Test
+ public void testPaillierGivenAllParameters() throws Exception
+ {
+ logger.info("Starting testPaillierGivenAllParameters: ");
+
+ Paillier pailler = new Paillier(p, q, bitLength);
+
+ assertEquals(pailler.getN(), N);
+ assertEquals(pailler.getLambdaN(), lambdaN);
+
+ // Check encryption
+ BigInteger encM1 = pailler.encrypt(m1, r1);
+ BigInteger encM2 = pailler.encrypt(m2, r2);
+ logger.info("encM1 = " + encM1.intValue() + " encM2 = " + encM2.intValue());
+
+ assertEquals(encM1, BigInteger.valueOf(14019));
+ assertEquals(encM2, BigInteger.valueOf(8836));
+
+ // Check decryption
+ BigInteger decM1 = pailler.decrypt(encM1);
+ BigInteger decM2 = pailler.decrypt(encM2);
+ logger.info("decM1 = " + decM1.intValue() + " decM2 = " + decM2.intValue());
+
+ assertEquals(decM1, m1);
+ assertEquals(decM2, m2);
+
+ // Check homomorphic property: E_r1(m1)*E_r2(m2) mod N^2 = E_r1r2((m1+m2) mod N) mod N^2
+ BigInteger encM1_times_encM2 = (encM1.multiply(encM2)).mod(NSquared);
+ BigInteger encM1plusM2 = pailler.encrypt((m1.add(m2)).mod(N), r1.multiply(r2));
+ logger.info("encM1_times_encM2 = " + encM1_times_encM2.intValue() + " encM1plusM2 = " + encM1plusM2.intValue());
+
+ assertEquals(encM1_times_encM2, BigInteger.valueOf(5617));
+ assertEquals(encM1plusM2, BigInteger.valueOf(5617));
+
+ logger.info("Successfully completed testPaillierGivenAllParameters: ");
+ }
+
+ @Test
+ public void testPaillierWithKeyGeneration() throws Exception
+ {
+ logger.info("Starting testPaillierWithKeyGeneration: ");
+
+ // Test with and without gmp optimization for modPow
+ SystemConfiguration.setProperty("pallier.FIPSPrimeGenerationChecks", "true");
+ SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
+ SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "true");
+ testPaillerWithKeyGenerationGeneral();
+
+ SystemConfiguration.setProperty("pallier.FIPSPrimeGenerationChecks", "false");
+
+ SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
+ SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "true");
+ testPaillerWithKeyGenerationGeneral();
+
+ SystemConfiguration.setProperty("paillier.useGMPForModPow", "true");
+ SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "false");
+ testPaillerWithKeyGenerationGeneral();
+
+ SystemConfiguration.setProperty("paillier.useGMPForModPow", "false");
+ SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "false");
+ testPaillerWithKeyGenerationGeneral();
+
+ // Reset the properties
+ SystemConfiguration.resetProperties();
+
+ logger.info("Ending testPaillierWithKeyGeneration: ");
+ }
+
+ public void testPaillerWithKeyGenerationGeneral() throws Exception
+ {
+ // Test without requiring highest bit to be set
+ logger.info("Starting testPaillierWithKeyGenerationBitSetOption with ensureHighBitSet = false");
+ testPaillierWithKeyGenerationBitSetOption(-1);
+
+ // Test requiring highest bit to be set
+ logger.info("Starting testPaillierWithKeyGenerationBitSetOption with ensureHighBitSet = true");
+ testPaillierWithKeyGenerationBitSetOption(5);
+ }
+
+ public void testPaillierWithKeyGenerationBitSetOption(int ensureBitSet) throws Exception
+ {
+ Random r = new Random();
+ int lowBitLength = 3073; // inclusive
+ int highBitLength = 7001; // exclusive
+
+ int loopVal = 1; // int loopVal = 1000; //change this and re-test for high loop testing
+ for (int i = 0; i < loopVal; ++i)
+ {
+ logger.info("i = " + i);
+
+ basicTestPaillierWithKeyGeneration(bitLength, certainty, ensureBitSet);
+ basicTestPaillierWithKeyGeneration(3072, certainty, ensureBitSet);
+
+ // Test with random bit length between 3073 and 7000
+ int randomLargeBitLength = r.nextInt(highBitLength - lowBitLength) + lowBitLength;
+ basicTestPaillierWithKeyGeneration(randomLargeBitLength, certainty, ensureBitSet);
+ }
+ }
+
+ private void basicTestPaillierWithKeyGeneration(int bitLengthInput, int certaintyInput, int ensureBitSet) throws Exception
+ {
+ Paillier pailler = new Paillier(bitLengthInput, certaintyInput, ensureBitSet);
+ BigInteger generatedN = pailler.getN();
+ BigInteger geneartedNsquared = generatedN.multiply(generatedN);
+
+ // Check the decrypting the encryption yields the message
+ BigInteger encM1 = pailler.encrypt(m1);
+ BigInteger encM2 = pailler.encrypt(m2);
+ logger.info("encM1 = " + encM1.intValue() + " encM2 = " + encM2.intValue());
+
+ BigInteger decM1 = pailler.decrypt(encM1);
+ BigInteger decM2 = pailler.decrypt(encM2);
+ logger.info("decM1 = " + decM1.intValue() + " decM2 = " + decM2.intValue());
+
+ assertEquals(decM1, m1);
+ assertEquals(decM2, m2);
+
+ // Check homomorphic property: E_r1(m1)*E_r2(m2) mod N^2 = E_r1r2((m1+m2) mod N) mod N^2
+ BigInteger encM1_times_encM2 = (encM1.multiply(encM2)).mod(geneartedNsquared);
+ BigInteger multDecrypt = pailler.decrypt(encM1_times_encM2);
+ BigInteger m1_plus_m2 = (m1.add(m2)).mod(N);
+
+ logger.info("encM1_times_encM2 = " + encM1_times_encM2.intValue() + " multDecrypt = " + multDecrypt.intValue() + " m1_plus_m2 = " + m1_plus_m2.intValue());
+
+ assertEquals(multDecrypt, m1_plus_m2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/0912bb65/src/test/java/org/apache/pirk/general/PartitionUtilsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/general/PartitionUtilsTest.java b/src/test/java/org/apache/pirk/general/PartitionUtilsTest.java
new file mode 100644
index 0000000..e20215b
--- /dev/null
+++ b/src/test/java/org/apache/pirk/general/PartitionUtilsTest.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.general;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
+
+import org.apache.pirk.schema.data.partitioner.IPDataPartitioner;
+import org.apache.pirk.schema.data.partitioner.ISO8601DatePartitioner;
+import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class to functionally test the bit conversion utils
+ */
+public class PartitionUtilsTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(PartitionUtilsTest.class);
+
+ @Test
+ public void testMask()
+ {
+ logger.info("Starting testMask: ");
+
+ BigInteger mask = PrimitiveTypePartitioner.formBitMask(4); // 1111
+
+ assertEquals(mask.intValue(), 15);
+
+ logger.info("Successfully completed testMask");
+ }
+
+ @Test
+ public void testPartitionBits()
+ {
+ logger.info("Starting testPartitionBits: ");
+
+ BigInteger value = new BigInteger("245"); // 11110101
+ BigInteger value2 = new BigInteger("983"); // 1111010111
+
+ BigInteger mask4 = PrimitiveTypePartitioner.formBitMask(4); // 1111
+ BigInteger mask8 = PrimitiveTypePartitioner.formBitMask(8); // 11111111
+
+ try
+ {
+ ArrayList<BigInteger> partitions = PrimitiveTypePartitioner.partitionBits(value, 4, mask4);
+
+ assertEquals(2, partitions.size());
+ assertEquals(partitions.get(0).intValue(), 15); // 1111
+ assertEquals(partitions.get(1).intValue(), 5); // 0101
+
+ } catch (Exception e)
+ {
+ fail(e.toString());
+ }
+
+ try
+ {
+ ArrayList<BigInteger> partitions = PrimitiveTypePartitioner.partitionBits(value2, 4, mask4);
+
+ assertEquals(3, partitions.size());
+ assertEquals(partitions.get(0).intValue(), 15); // 1111
+ assertEquals(partitions.get(1).intValue(), 5); // 0101
+ assertEquals(partitions.get(2).intValue(), 3); // 11
+
+ } catch (Exception e)
+ {
+ fail(e.toString());
+ }
+ try
+ {
+ ArrayList<BigInteger> partitions = PrimitiveTypePartitioner.partitionBits(value, 8, mask8);
+
+ assertEquals(1, partitions.size());
+ assertEquals(partitions.get(0).intValue(), 245);
+
+ } catch (Exception e)
+ {
+ fail(e.toString());
+ }
+
+ try
+ {
+ ArrayList<BigInteger> partitions = PrimitiveTypePartitioner.partitionBits(value, 4, mask8);
+
+ fail("BitConversionUtils.partitionBits did not throw error for mismatched partitionSize and mask size");
+
+ } catch (Exception ignore)
+ {}
+
+ logger.info("Successfully completed testPartitionBits");
+ }
+
+ @Test
+ public void testPartitions() throws Exception
+ {
+ logger.info("Starting testToPartitions:");
+
+ PrimitiveTypePartitioner primitivePartitioner = new PrimitiveTypePartitioner();
+ IPDataPartitioner ipPartitioner = new IPDataPartitioner();
+ ISO8601DatePartitioner datePartitioner = new ISO8601DatePartitioner();
+
+ // Test IP
+ String ipTest = "127.0.0.1";
+ ArrayList<BigInteger> partsIP = ipPartitioner.toPartitions(ipTest, PrimitiveTypePartitioner.STRING);
+ assertEquals(4, partsIP.size());
+ assertEquals(ipTest, ipPartitioner.fromPartitions(partsIP, 0, PrimitiveTypePartitioner.STRING));
+
+ // Test Date
+ String dateTest = "2016-02-20T23:29:05.000Z";
+ ArrayList<BigInteger> partsDate = datePartitioner.toPartitions(dateTest, null);
+ assertEquals(8, partsDate.size());
+ assertEquals(dateTest, datePartitioner.fromPartitions(partsDate, 0, null));
+
+ // Test byte
+ byte bTest = Byte.parseByte("10");
+ ArrayList<BigInteger> partsByte = primitivePartitioner.toPartitions(bTest, PrimitiveTypePartitioner.BYTE);
+ assertEquals(1, partsByte.size());
+ assertEquals(bTest, primitivePartitioner.fromPartitions(partsByte, 0, PrimitiveTypePartitioner.BYTE));
+
+ ArrayList<BigInteger> partsByteMax = primitivePartitioner.toPartitions(Byte.MAX_VALUE, PrimitiveTypePartitioner.BYTE);
+ assertEquals(1, partsByteMax.size());
+ assertEquals(Byte.MAX_VALUE, primitivePartitioner.fromPartitions(partsByteMax, 0, PrimitiveTypePartitioner.BYTE));
+
+ // Test string
+ String stringBits = SystemConfiguration.getProperty("pir.stringBits");
+ SystemConfiguration.setProperty("pir.stringBits", "64");
+ testString("testString"); // over the allowed bit size
+ testString("t"); // under the allowed bit size
+ SystemConfiguration.setProperty("pir.stringBits", stringBits);
+
+ // Test short
+ short shortTest = new Short("2456");
+ ArrayList<BigInteger> partsShort = primitivePartitioner.toPartitions(shortTest, PrimitiveTypePartitioner.SHORT);
+ assertEquals(2, partsShort.size());
+ assertEquals(shortTest, primitivePartitioner.fromPartitions(partsShort, 0, PrimitiveTypePartitioner.SHORT));
+
+ ArrayList<BigInteger> partsShortMax = primitivePartitioner.toPartitions(Short.MAX_VALUE, PrimitiveTypePartitioner.SHORT);
+ assertEquals(2, partsShortMax.size());
+ assertEquals(Short.MAX_VALUE, primitivePartitioner.fromPartitions(partsShortMax, 0, PrimitiveTypePartitioner.SHORT));
+
+ // Test int
+ int intTest = Integer.parseInt("-5789");
+ ArrayList<BigInteger> partsInt = primitivePartitioner.toPartitions(intTest, PrimitiveTypePartitioner.INT);
+ assertEquals(4, partsInt.size());
+ assertEquals(intTest, primitivePartitioner.fromPartitions(partsInt, 0, PrimitiveTypePartitioner.INT));
+
+ ArrayList<BigInteger> partsIntMax = primitivePartitioner.toPartitions(Integer.MAX_VALUE, PrimitiveTypePartitioner.INT);
+ assertEquals(4, partsIntMax.size());
+ assertEquals(Integer.MAX_VALUE, primitivePartitioner.fromPartitions(partsIntMax, 0, PrimitiveTypePartitioner.INT));
+
+ // Test long
+ long longTest = Long.parseLong("56789");
+ ArrayList<BigInteger> partsLong = primitivePartitioner.toPartitions(longTest, PrimitiveTypePartitioner.LONG);
+ assertEquals(8, partsLong.size());
+ assertEquals(longTest, primitivePartitioner.fromPartitions(partsLong, 0, PrimitiveTypePartitioner.LONG));
+
+ ArrayList<BigInteger> partsLongMax = primitivePartitioner.toPartitions(Long.MAX_VALUE, PrimitiveTypePartitioner.LONG);
+ assertEquals(8, partsLongMax.size());
+ assertEquals(Long.MAX_VALUE, primitivePartitioner.fromPartitions(partsLongMax, 0, PrimitiveTypePartitioner.LONG));
+
+ // Test float
+ float floatTest = Float.parseFloat("567.77");
+ ArrayList<BigInteger> partsFloat = primitivePartitioner.toPartitions(floatTest, PrimitiveTypePartitioner.FLOAT);
+ assertEquals(4, partsFloat.size());
+ assertEquals(floatTest, primitivePartitioner.fromPartitions(partsFloat, 0, PrimitiveTypePartitioner.FLOAT));
+
+ ArrayList<BigInteger> partsFloatMax = primitivePartitioner.toPartitions(Float.MAX_VALUE, PrimitiveTypePartitioner.FLOAT);
+ assertEquals(4, partsFloatMax.size());
+ assertEquals(Float.MAX_VALUE, primitivePartitioner.fromPartitions(partsFloatMax, 0, PrimitiveTypePartitioner.FLOAT));
+
+ // Test double
+ double doubleTest = Double.parseDouble("567.77");
+ ArrayList<BigInteger> partsDouble = primitivePartitioner.toPartitions(doubleTest, PrimitiveTypePartitioner.DOUBLE);
+ assertEquals(8, partsDouble.size());
+ assertEquals(doubleTest, primitivePartitioner.fromPartitions(partsDouble, 0, PrimitiveTypePartitioner.DOUBLE));
+
+ ArrayList<BigInteger> partsDoubleMax = primitivePartitioner.toPartitions(Double.MAX_VALUE, PrimitiveTypePartitioner.DOUBLE);
+ assertEquals(8, partsDoubleMax.size());
+ assertEquals(Double.MAX_VALUE, primitivePartitioner.fromPartitions(partsDoubleMax, 0, PrimitiveTypePartitioner.DOUBLE));
+
+ // Test char
+ char charTest = 'b';
+ ArrayList<BigInteger> partsChar = primitivePartitioner.toPartitions(charTest, PrimitiveTypePartitioner.CHAR);
+ assertEquals(2, partsChar.size());
+ assertEquals(charTest, primitivePartitioner.fromPartitions(partsChar, 0, PrimitiveTypePartitioner.CHAR));
+
+ ArrayList<BigInteger> partsCharMax = primitivePartitioner.toPartitions(Character.MAX_VALUE, PrimitiveTypePartitioner.CHAR);
+ assertEquals(2, partsCharMax.size());
+ assertEquals(Character.MAX_VALUE, primitivePartitioner.fromPartitions(partsCharMax, 0, PrimitiveTypePartitioner.CHAR));
+
+ logger.info("Sucessfully completed testToPartitions:");
+ }
+
+ private void testString(String testString) throws Exception
+ {
+ PrimitiveTypePartitioner ptp = new PrimitiveTypePartitioner();
+
+ ArrayList<BigInteger> partsString = ptp.toPartitions(testString, PrimitiveTypePartitioner.STRING);
+ int numParts = Integer.parseInt(SystemConfiguration.getProperty("pir.stringBits")) / 8;
+ assertEquals(numParts, partsString.size());
+
+ logger.info("testString.getBytes().length = " + testString.getBytes().length);
+ int offset = numParts;
+ if (testString.getBytes().length < numParts)
+ {
+ offset = testString.getBytes().length;
+ }
+ String element = new String(testString.getBytes(), 0, offset);
+ assertEquals(element, ptp.fromPartitions(partsString, 0, PrimitiveTypePartitioner.STRING));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/0912bb65/src/test/java/org/apache/pirk/general/QueryParserUtilsTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/general/QueryParserUtilsTest.java b/src/test/java/org/apache/pirk/general/QueryParserUtilsTest.java
new file mode 100644
index 0000000..9ac2522
--- /dev/null
+++ b/src/test/java/org/apache/pirk/general/QueryParserUtilsTest.java
@@ -0,0 +1,421 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.general;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.apache.hadoop.io.MapWritable;
+import org.apache.pirk.schema.data.DataSchema;
+import org.apache.pirk.schema.data.DataSchemaRegistry;
+import org.apache.pirk.schema.query.QuerySchemaRegistry;
+import org.apache.pirk.test.utils.Inputs;
+import org.apache.pirk.utils.QueryParserUtils;
+import org.apache.pirk.utils.StringUtils;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.json.simple.JSONObject;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class for testing the QueryParser methods
+ */
+public class QueryParserUtilsTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(QueryParserUtilsTest.class);
+
+ private static MapWritable doc = null; // MapWritable with arrays in json string representation
+ private static MapWritable docWAW = null; // MapWritable with arrays as WritableArrayWritable objects
+ private static Map<String,Object> docMap = null; // arrays as ArrayList<String>
+
+ private static DataSchema dSchema = null;
+
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ ArrayList<JSONObject> dataElementsJSON = Inputs.createJSONDataElements();
+
+ // Reset the schema properties and registries
+ DataSchemaRegistry.clearRegistry();
+ QuerySchemaRegistry.clearRegistry();
+ SystemConfiguration.setProperty("data.schemas", "none");
+ SystemConfiguration.setProperty("query.schemas", "none");
+
+ Inputs.createSchemaFiles(null, false, null);
+
+ dSchema = DataSchemaRegistry.get(Inputs.TEST_DATA_SCHEMA_NAME);
+
+ // ProcessBuilder pAdd1 = new ProcessBuilder("curl", "-XPUT", indexTypeNum1, "-d",
+ // "{\"qname\":\"a.b.c.com\",\"date\":\"2016-02-20T23:29:05.000Z\",\"qtype\":[\"1\"]"
+ // + ",\"rcode\":\"0\",\"src_ip\":\"55.55.55.55\",\"dest_ip\":\"1.2.3.6\"" + ",\"ip\":[\"10.20.30.40\",\"10.20.30.60\"]}");
+ //
+ doc = StringUtils.jsonStringToMapWritableWithArrayWritable(dataElementsJSON.get(0).toJSONString(), dSchema);
+ docWAW = StringUtils.jsonStringToMapWritableWithWritableArrayWritable(dataElementsJSON.get(0).toJSONString(), dSchema);
+ docMap = StringUtils.jsonStringToMap(dataElementsJSON.get(0).toJSONString(), dSchema);
+ }
+
+ @AfterClass
+ public static void teardown()
+ {
+ // Reset the schema properties and registries
+ DataSchemaRegistry.clearRegistry();
+ QuerySchemaRegistry.clearRegistry();
+ SystemConfiguration.setProperty("data.schemas", "none");
+ SystemConfiguration.setProperty("query.schemas", "none");
+ }
+
+ @Test
+ public void testSingleQuery()
+ {
+ String query1 = "?q=src_ip:55.55.55.55";
+ assertTrue(QueryParserUtils.checkRecord(query1, doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query1, docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecord(query1, docMap, dSchema));
+
+ String query2 = "?q=qname:a.b.c.com";
+ assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema));
+
+ String query3 = "?q=qname:d.b.c.com";
+ assertFalse(QueryParserUtils.checkRecord(query3, doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecord(query3, docMap, dSchema));
+ }
+
+ @Test
+ public void testQueryFieldDoesNotExist()
+ {
+ logger.info("running testQueryFieldDoesNotExist");
+
+ // Field does not exist, this should not be found
+ String query = "?q=nonexistent-field:*check*";
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query, docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecord(query, doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord(query, docMap, dSchema));
+
+ // First field does not exist, but second should be found
+ String query2 = "?q=nonexistent-field:*check*+OR+qname:*a.b.c.com*";
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema));
+
+ // First field does not exist, second field does, but AND operator makes query false
+ String query3 = "?q=nonexistent-field:*check*+AND+qname:*a.b.c.com*";
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecord(query3, doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord(query3, docMap, dSchema));
+
+ logger.info("completed testQueryFieldDoesNotExist");
+ }
+
+ @Test
+ public void testIgnoreCase()
+ {
+ logger.info("running testIgnoreCase");
+
+ // with case sensitivity, should NOT be found
+ String query = "?q=qname:*A.b.c.com*";
+ assertFalse(QueryParserUtils.checkRecord(query, doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query, docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecord(query, docMap, dSchema));
+
+ // with case sensitivity, should be found
+ String query2 = "?q=qname:*a.b.c.com*";
+ assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema));
+
+ // adds @ flag = case insensitivity, thus should be found
+ String query3 = "?q=qname@:*A.b.c.com*";
+ assertTrue(QueryParserUtils.checkRecord(query3, doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecord(query3, docMap, dSchema));
+
+ logger.info("completed testIgnoreCase");
+ }
+
+ @Test
+ public void testSingleValueRangeQuery()
+ {
+ testSingleValueRangeQueryMapWritable();
+ testSingleValueRangeQueryMap();
+ testSingleValueRangeQueryMapWritableWAW();
+ }
+
+ private void testSingleValueRangeQueryMapWritable()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=rcode:[0+TO+2]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=rcode:{-1+TO+2}", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=rcode:[-1+TO+0]", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=rcode:{0+TO+3}", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=rcode:[3+TO+10]", doc, dSchema));
+ }
+
+ private void testSingleValueRangeQueryMap()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=rcode:[0+TO+2]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=rcode:{-1+TO+2}", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=rcode:[-1+TO+0]", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=rcode:{0+TO+3}", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=rcode:[3+TO+10]", docMap, dSchema));
+ }
+
+ private void testSingleValueRangeQueryMapWritableWAW()
+ {
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[0+TO+2]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:{-1+TO+2}", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[-1+TO+0]", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:{0+TO+3}", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[3+TO+10]", docWAW, dSchema));
+ }
+
+ @Test
+ public void testIPRangeQuery()
+ {
+ testIPRangeQueryMapWritable();
+ testIPRangeQueryMap();
+ testIPRangeQueryMapWritableWAW();
+ }
+
+ public void testIPRangeQueryMapWritable()
+ {
+ // src_ip: 55.55.55.55
+ // ip: 10.20.30.40,10.20.30.60
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", doc, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", doc, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=ip:[10.20.30.50+TO+10.20.30.69]", doc, dSchema));
+ }
+
+ public void testIPRangeQueryMapWritableWAW()
+ {
+ // src_ip: 55.55.55.55
+ // ip: 10.20.30.40,10.20.30.60
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", docWAW, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", docWAW, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=ip:[10.20.30.50+TO+10.20.30.69]", docWAW, dSchema));
+ }
+
+ public void testIPRangeQueryMap()
+ {
+ // src_ip: 55.55.55.55
+ // ip: 10.20.30.40,10.20.30.60
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", docMap, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", docMap, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=ip:[10.20.30.50+TO+10.20.30.69]", docMap, dSchema));
+ }
+
+ @Test
+ public void testDateRangeQuery()
+ {
+ testDateRangeQueryMapWritable();
+ testDateRangeQueryMapWritableWAW();
+ testDateRangeQueryMap();
+ }
+
+ private void testDateRangeQueryMapWritable()
+ {
+ // date: 2016-02-20T23:29:05.000Z
+
+ assertTrue(QueryParserUtils.checkRecord("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", doc, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", doc, dSchema));
+ }
+
+ private void testDateRangeQueryMap()
+ {
+ // date: 2016-02-20T23:29:05.000Z
+
+ assertTrue(QueryParserUtils.checkRecord("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", docMap, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", docMap, dSchema));
+ }
+
+ private void testDateRangeQueryMapWritableWAW()
+ {
+ // date: 2016-02-20T23:29:05.000Z
+
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", docWAW, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", docWAW, dSchema));
+ }
+
+ @Test
+ public void testBooleanQuery()
+ {
+ testBooleanQueryMapWritable();
+ testBooleanQueryMapMapWritableWAW();
+ testBooleanQueryMap();
+ }
+
+ private void testBooleanQueryMapWritable()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+qtype:2+OR+rcode:0", doc, dSchema));
+ }
+
+ private void testBooleanQueryMap()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+qtype:2+OR+rcode:0", docMap, dSchema));
+ }
+
+ private void testBooleanQueryMapMapWritableWAW()
+ {
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]",
+ docWAW, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]",
+ docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]",
+ docWAW, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:5+OR+qtype:2+OR+rcode:0", docWAW, dSchema));
+ }
+
+ @Test
+ public void testAllQuery()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=*", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=*", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=*", docWAW, dSchema));
+ }
+
+ @Test
+ public void testWildcardQuery()
+ {
+ testWildcardQueryMapWritable();
+ testWildcardQueryMap();
+ testWildcardQueryMapWritableWAW();
+ }
+
+ private void testWildcardQueryMapWritable()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:*.com", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c*m", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b*", doc, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:*.org", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:mrtf*", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljnik*.uk", doc, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c?m", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.?.com", doc, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:?.b.c.com", doc, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:medelj?ikafera.com", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljntkafer?.com", doc, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:?edeljnikrfera.com", doc, dSchema));
+ }
+
+ private void testWildcardQueryMap()
+ {
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:*.com", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c*m", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b*", docMap, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:*.org", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:mrtf*", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljnik*.uk", docMap, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c?m", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.?.com", docMap, dSchema));
+ assertTrue(QueryParserUtils.checkRecord("?q=qname:?.b.c.com", docMap, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:medelj?ikafera.com", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljntkafer?.com", docMap, dSchema));
+ assertFalse(QueryParserUtils.checkRecord("?q=qname:?edeljnikrfera.com", docMap, dSchema));
+ }
+
+ private void testWildcardQueryMapWritableWAW()
+ {
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:*.com", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.c.c*m", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b*", docWAW, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:*.org", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:mrtf*", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:nedeljnik*.uk", docWAW, dSchema));
+
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.c.c?m", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.?.com", docWAW, dSchema));
+ assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:?.b.c.com", docWAW, dSchema));
+
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:medelj?ikafera.com", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:nedeljntkafer?.com", docWAW, dSchema));
+ assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:?edeljnikrfera.com", docWAW, dSchema));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/0912bb65/src/test/java/org/apache/pirk/schema/data/LoadDataSchemaTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/schema/data/LoadDataSchemaTest.java b/src/test/java/org/apache/pirk/schema/data/LoadDataSchemaTest.java
new file mode 100644
index 0000000..3aa500b
--- /dev/null
+++ b/src/test/java/org/apache/pirk/schema/data/LoadDataSchemaTest.java
@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.schema.data;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.apache.pirk.schema.data.partitioner.IPDataPartitioner;
+import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner;
+import org.apache.pirk.test.utils.TestUtils;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+/**
+ * Test suite for LoadDataSchema and DataSchema
+ */
+public class LoadDataSchemaTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(LoadDataSchemaTest.class);
+
+ private String dataSchemaName = "fakeDataSchema";
+
+ private String element1 = "elementName1";
+ private String element2 = "elementName2";
+ private String element3 = "elementName3";
+
+ @Test
+ public void testGeneralSchemaLoad() throws Exception
+ {
+ // Pull off the property and reset upon completion
+ String schemasProp = SystemConfiguration.getProperty("data.schemas", "none");
+
+ // Write the schema file
+ try
+ {
+ createDataSchema("schemaFile");
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+
+ // Force the schema to load
+ DataSchemaLoader.initialize();
+
+ // Check the entries
+ DataSchema dSchema = DataSchemaRegistry.get(dataSchemaName);
+
+ assertEquals(dataSchemaName, dSchema.getSchemaName());
+
+ assertEquals(3, dSchema.getElementNames().size());
+
+ // TODO: check Hadoop text names
+
+ assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element1));
+ assertEquals(PrimitiveTypePartitioner.INT, dSchema.getElementType(element2));
+ assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element3));
+
+ assertEquals(PrimitiveTypePartitioner.class.getName(), dSchema.getPartitionerTypeName(element1));
+ if (!(dSchema.getPartitionerForElement(element1) instanceof PrimitiveTypePartitioner))
+ {
+ fail("Partitioner instance for element1 must be PrimitiveTypePartitioner");
+ }
+ assertEquals(IPDataPartitioner.class.getName(), dSchema.getPartitionerTypeName(element3));
+ if (!(dSchema.getPartitionerForElement(element3) instanceof IPDataPartitioner))
+ {
+ fail("Partitioner instance for element3 must be IPDataPartitioner");
+ }
+
+ assertEquals(2, dSchema.getArrayElements().size());
+ assertTrue(dSchema.getArrayElements().contains(element2));
+ assertTrue(dSchema.getArrayElements().contains(element3));
+
+ assertEquals(1, dSchema.getNonArrayElements().size());
+ assertTrue(dSchema.getNonArrayElements().contains(element1));
+
+ // Reset original data.schemas property
+ SystemConfiguration.setProperty("data.schemas", schemasProp);
+
+ // Force the schema to load
+ if (!schemasProp.equals("none"))
+ {
+ DataSchemaLoader.initialize();
+ }
+ }
+
+ @Test
+ public void testIncorrectJavaType() throws Exception
+ {
+ // Pull off the property and reset upon completion
+ String schemasProp = SystemConfiguration.getProperty("data.schemas");
+
+ // Write the schema file
+ try
+ {
+ createDataSchemaIncorrectJavaType("wrongJavaType");
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+
+ try
+ {
+ // Force the schema to load
+ DataSchemaLoader.initialize();
+ fail("DataSchemaLoader did not throw exception for incorrect javaType");
+ } catch (Exception ignore)
+ {}
+
+ // Reset original data.schemas property
+ SystemConfiguration.setProperty("data.schemas", schemasProp);
+
+ // Force the schema to load
+ DataSchemaLoader.initialize();
+ }
+
+ @Test
+ public void testUnknownPartitioner() throws Exception
+ {
+ // Pull off the property and reset upon completion
+ String schemasProp = SystemConfiguration.getProperty("data.schemas");
+
+ // Write the schema file
+ try
+ {
+ createDataSchemaUnknownPartitioner("unknownPartitioner");
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+
+ try
+ {
+ // Force the schema to load
+ DataSchemaLoader.initialize();
+ fail("DataSchemaLoader did not throw exception for unknown partitioner");
+ } catch (Exception ignore)
+ {}
+
+ // Reset original data.schemas property
+ SystemConfiguration.setProperty("data.schemas", schemasProp);
+
+ // Force the schema to load
+ DataSchemaLoader.initialize();
+ }
+
+ // Create the file that contains an unknown partitioner
+ private void createDataSchemaUnknownPartitioner(String schemaFile) throws IOException
+ {
+ // Create a temporary file for the test schema, set in the properties
+ File file = File.createTempFile(schemaFile, ".xml");
+ file.deleteOnExit();
+ logger.info("file = " + file.toString());
+ SystemConfiguration.setProperty("data.schemas", file.toString());
+
+ // Write to the file
+ try
+ {
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+ Document doc = dBuilder.newDocument();
+
+ // root element
+ Element rootElement = doc.createElement("schema");
+ doc.appendChild(rootElement);
+
+ // Add the schemaName
+ Element schemaNameElement = doc.createElement("schemaName");
+ schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
+ rootElement.appendChild(schemaNameElement);
+
+ // Add the element - unknown partitioner
+ TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.INT, "false", "fakePartitioner");
+
+ // Write to a xml file
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ Transformer transformer = transformerFactory.newTransformer();
+ DOMSource source = new DOMSource(doc);
+ StreamResult result = new StreamResult(file);
+ transformer.transform(source, result);
+
+ // Output for testing
+ StreamResult consoleResult = new StreamResult(System.out);
+ transformer.transform(source, consoleResult);
+ System.out.println();
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ }
+ }
+
+ // Create the test data schema file
+ private void createDataSchema(String schemaFile) throws IOException
+ {
+ // Create a temporary file for the test schema, set in the properties
+ File file = File.createTempFile(schemaFile, ".xml");
+ file.deleteOnExit();
+ logger.info("file = " + file.toString());
+ SystemConfiguration.setProperty("data.schemas", file.toString());
+
+ // Write to the file
+ try
+ {
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+ Document doc = dBuilder.newDocument();
+
+ // root element
+ Element rootElement = doc.createElement("schema");
+ doc.appendChild(rootElement);
+
+ // Add the schemaName
+ Element schemaNameElement = doc.createElement("schemaName");
+ schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
+ rootElement.appendChild(schemaNameElement);
+
+ // Add the elements
+ // element1 -- single String
+ // TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.STRING, "false", PrimitiveTypePartitioner.class.getName());
+ TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.STRING, "false", null);
+
+ // element2 - -- array of Integers
+ TestUtils.addElement(doc, rootElement, element2, PrimitiveTypePartitioner.INT, "true", PrimitiveTypePartitioner.class.getName());
+
+ // element3 -- array of IP addresses
+ TestUtils.addElement(doc, rootElement, element3, PrimitiveTypePartitioner.STRING, "true", IPDataPartitioner.class.getName());
+
+ // Write to a xml file
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ Transformer transformer = transformerFactory.newTransformer();
+ DOMSource source = new DOMSource(doc);
+ StreamResult result = new StreamResult(file);
+ transformer.transform(source, result);
+
+ // Output for testing
+ StreamResult consoleResult = new StreamResult(System.out);
+ transformer.transform(source, consoleResult);
+ System.out.println();
+
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ }
+ }
+
+ // Create the test schema file
+ private void createDataSchemaIncorrectJavaType(String schemaFile) throws IOException
+ {
+ // Create a temporary file for the test schema, set in the properties
+ File file = File.createTempFile(schemaFile, ".xml");
+ file.deleteOnExit();
+ logger.info("file = " + file.toString());
+ SystemConfiguration.setProperty("data.schemas", file.toString());
+
+ // Write to the file
+ try
+ {
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+ Document doc = dBuilder.newDocument();
+
+ // root element
+ Element rootElement = doc.createElement("schema");
+ doc.appendChild(rootElement);
+
+ // Add the schemaName
+ Element schemaNameElement = doc.createElement("schemaName");
+ schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
+ rootElement.appendChild(schemaNameElement);
+
+ // Add the element - unknown Java type
+ TestUtils.addElement(doc, rootElement, element1, "bogus", "false", PrimitiveTypePartitioner.class.getName());
+
+ // Write to a xml file
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ Transformer transformer = transformerFactory.newTransformer();
+ DOMSource source = new DOMSource(doc);
+ StreamResult result = new StreamResult(file);
+ transformer.transform(source, result);
+
+ // Output for testing
+ StreamResult consoleResult = new StreamResult(System.out);
+ transformer.transform(source, consoleResult);
+ System.out.println();
+
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/0912bb65/src/test/java/org/apache/pirk/schema/query/LoadQuerySchemaTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/schema/query/LoadQuerySchemaTest.java b/src/test/java/org/apache/pirk/schema/query/LoadQuerySchemaTest.java
new file mode 100644
index 0000000..55cb0a9
--- /dev/null
+++ b/src/test/java/org/apache/pirk/schema/query/LoadQuerySchemaTest.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.schema.query;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.apache.pirk.schema.data.DataSchemaLoader;
+import org.apache.pirk.schema.data.partitioner.IPDataPartitioner;
+import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner;
+import org.apache.pirk.schema.query.filter.StopListFilter;
+import org.apache.pirk.test.utils.Inputs;
+import org.apache.pirk.test.utils.TestUtils;
+import org.apache.pirk.utils.PIRException;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+/**
+ * Test suite for LoadQuerySchema and QuerySchema
+ */
+public class LoadQuerySchemaTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(LoadQuerySchemaTest.class);
+
+ private String querySchemaFile = "querySchemaFile";
+ private String dataSchemaName = "fakeDataSchema";
+ private String querySchemaName = "fakeQuerySchema";
+
+ private String element1 = "elementName1";
+ private String element2 = "elementName2";
+ private String element3 = "elementName3";
+ private String element4 = "elementName4";
+
+ private List<String> queryElements = Arrays.asList(element1, element2, element3);
+ private List<String> filterElements = Collections.singletonList(element2);
+
+ @Test
+ public void testGeneralSchemaLoad() throws Exception
+ {
+ logger.info("Starting testGeneralSchemaLoad: ");
+
+ // Pull off the properties and reset upon completion
+ String dataSchemasProp = SystemConfiguration.getProperty("data.schemas", "none");
+ String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
+ String stopListFileProp = SystemConfiguration.getProperty("pir.stopListFile");
+
+ // Create the stoplist file
+ createStopListFile();
+
+ // Create the data schema used and force it to load
+ try
+ {
+ createDataSchema("dataSchemaFile");
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ DataSchemaLoader.initialize();
+
+ // Create the query schema used and force it to load
+ try
+ {
+ TestUtils.createQuerySchema(querySchemaFile, querySchemaName, dataSchemaName, element4, queryElements, filterElements, StopListFilter.class.getName());
+
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ QuerySchemaLoader.initialize();
+
+ // Check the entries
+ QuerySchema qSchema = QuerySchemaRegistry.get(querySchemaName);
+
+ assertEquals(querySchemaName, qSchema.getSchemaName());
+ assertEquals(dataSchemaName, qSchema.getDataSchemaName());
+ assertEquals(element4, qSchema.getSelectorName());
+
+ assertEquals(StopListFilter.class.getName(), qSchema.getFilterTypeName());
+ if (!(qSchema.getFilter() instanceof StopListFilter))
+ {
+ fail("Filter class instance must be StopListFilter");
+ }
+
+ assertEquals(3, qSchema.getElementNames().size());
+ for (String item : qSchema.getElementNames())
+ {
+ if (!(item.equals(element1) || item.equals(element2) || item.equals(element3)))
+ {
+ fail("elementNames: item = " + item + " must equal one of: " + element1 + ", " + element2 + ", or " + element3);
+ }
+ }
+ assertEquals(1, qSchema.getFilteredElementNames().size());
+ for (String item : qSchema.getFilteredElementNames())
+ {
+ if (!item.equals(element2))
+ {
+ fail("filterElementNames: item = " + item + " must equal " + element2);
+ }
+ }
+
+ // one string, array IPs, array integers
+ int stringSize = Integer.parseInt(SystemConfiguration.getProperty("pir.stringBits"));
+ int arrayMult = Integer.parseInt(SystemConfiguration.getProperty("pir.numReturnArrayElements"));
+ int dataElementSize = stringSize + 32 * arrayMult + 32 * arrayMult;
+ assertEquals(dataElementSize, qSchema.getDataElementSize());
+
+ // Reset original query and data schema properties
+ SystemConfiguration.setProperty("data.schemas", dataSchemasProp);
+ SystemConfiguration.setProperty("query.schemas", querySchemasProp);
+ SystemConfiguration.setProperty("pir.stopListFile", stopListFileProp);
+
+ // Force the query and data schemas to load their original values
+ if (!dataSchemasProp.equals("none"))
+ {
+ DataSchemaLoader.initialize();
+ }
+
+ if (!querySchemasProp.equals("none"))
+ {
+ QuerySchemaLoader.initialize();
+ }
+
+ logger.info("Finished testGeneralSchemaLoad: ");
+ }
+
+ @Test
+ public void testUnknownFilterClass() throws Exception
+ {
+ // Pull off the properties and reset upon completion
+ String dataSchemasProp = SystemConfiguration.getProperty("data.schemas", "none");
+ String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
+
+ // Create the data schema used and force it to load
+ try
+ {
+ createDataSchema("dataSchemaFile");
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ DataSchemaLoader.initialize();
+
+ // Create the query schema used and force it to load
+ try
+ {
+ TestUtils.createQuerySchema(querySchemaFile, querySchemaName, dataSchemaName, "nonExistentElement", queryElements, filterElements, "bogusFilterClass");
+
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ try
+ {
+ QuerySchemaLoader.initialize();
+ fail("QuerySchemaLoader did not throw exception for bogus filter class");
+ } catch (Exception ignore)
+ {}
+
+ // Reset original query and data schema properties
+ SystemConfiguration.setProperty("data.schemas", dataSchemasProp);
+ SystemConfiguration.setProperty("query.schemas", querySchemasProp);
+
+ // Force the query and data schemas to load their original values
+ if (!dataSchemasProp.equals("none"))
+ {
+ DataSchemaLoader.initialize();
+ }
+
+ if (!querySchemasProp.equals("none"))
+ {
+ QuerySchemaLoader.initialize();
+ }
+
+ logger.info("Finished testFunkyFilterScenarios");
+ }
+
+ @Test
+ public void testDataSchemaDoesNotExist() throws Exception
+ {
+ logger.info("Starting testDataSchemaDoesNotExist: ");
+
+ // Pull off the properties and reset upon completion
+ String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
+
+ // Create the query schema used and force it to load
+ try
+ {
+ TestUtils.createQuerySchema(querySchemaFile, querySchemaName, dataSchemaName, element4, queryElements, filterElements, null);
+
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ try
+ {
+ QuerySchemaLoader.initialize();
+ fail("QuerySchemaLoader did not throw exception for non-existent DataSchema");
+ } catch (Exception ignore)
+ {}
+
+ // Reset original query properties and force to load
+ SystemConfiguration.setProperty("query.schemas", querySchemasProp);
+ if (!querySchemasProp.equals("none"))
+ {
+ QuerySchemaLoader.initialize();
+ }
+
+ logger.info("Finished testDataSchemaDoesNotExist ");
+ }
+
+ @Test
+ public void testSelectorDoesNotExistInDataSchema() throws Exception
+ {
+ logger.info("Starting testSelectorDoesNotExistInDataSchema: ");
+
+ // Pull off the properties and reset upon completion
+ String dataSchemasProp = SystemConfiguration.getProperty("data.schemas", "none");
+ String querySchemasProp = SystemConfiguration.getProperty("query.schemas", "none");
+
+ // Create the data schema used and force it to load
+ try
+ {
+ createDataSchema("dataSchemaFile");
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ DataSchemaLoader.initialize();
+
+ // Create the query schema used and force it to load
+ try
+ {
+ TestUtils.createQuerySchema(querySchemaFile, querySchemaName, dataSchemaName, "nonExistentElement", queryElements, filterElements,
+ StopListFilter.class.getName());
+
+ } catch (IOException e)
+ {
+ e.printStackTrace();
+ fail(e.toString());
+ }
+ try
+ {
+ QuerySchemaLoader.initialize();
+ fail("QuerySchemaLoader did not throw exception for non-existent selectorName");
+ } catch (Exception ignore)
+ {}
+
+ // Reset original query and data schema properties
+ SystemConfiguration.setProperty("data.schemas", dataSchemasProp);
+ SystemConfiguration.setProperty("query.schemas", querySchemasProp);
+
+ // Force the query and data schemas to load their original values
+ if (!dataSchemasProp.equals("none"))
+ {
+ DataSchemaLoader.initialize();
+ }
+
+ if (!querySchemasProp.equals("none"))
+ {
+ QuerySchemaLoader.initialize();
+ }
+
+ logger.info("Finished testSelectorDoesNotExistInDataSchema ");
+ }
+
+ // Create the stoplist file and alter the properties accordingly
+ private void createStopListFile() throws IOException, PIRException
+ {
+ SystemConfiguration.setProperty("pir.stopListFile", "testStopListFile");
+ String newSLFile = Inputs.createPIRStopList(null, false);
+ SystemConfiguration.setProperty("pir.stopListFile", newSLFile);
+ }
+
+ // Create the test data schema file
+ private void createDataSchema(String schemaFile) throws IOException
+ {
+ // Create a temporary file for the test schema, set in the properties
+ File file = File.createTempFile(schemaFile, ".xml");
+ file.deleteOnExit();
+ logger.info("file = " + file.toString());
+ SystemConfiguration.setProperty("data.schemas", file.toString());
+
+ // Write to the file
+ try
+ {
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+ Document doc = dBuilder.newDocument();
+
+ // root element
+ Element rootElement = doc.createElement("schema");
+ doc.appendChild(rootElement);
+
+ // Add the schemaName
+ Element schemaNameElement = doc.createElement("schemaName");
+ schemaNameElement.appendChild(doc.createTextNode(dataSchemaName));
+ rootElement.appendChild(schemaNameElement);
+
+ // Add the elements
+ // element1 -- single String
+ TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.STRING, "false", PrimitiveTypePartitioner.class.getName());
+
+ // element2 - -- array of Integers
+ TestUtils.addElement(doc, rootElement, element2, PrimitiveTypePartitioner.INT, "true", PrimitiveTypePartitioner.class.getName());
+
+ // element3 -- array of IP addresses
+ TestUtils.addElement(doc, rootElement, element3, PrimitiveTypePartitioner.STRING, "true", IPDataPartitioner.class.getName());
+
+ // element4 -- single byte type
+ TestUtils.addElement(doc, rootElement, element4, PrimitiveTypePartitioner.BYTE, "false", PrimitiveTypePartitioner.class.getName());
+
+ // Write to a xml file
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ Transformer transformer = transformerFactory.newTransformer();
+ DOMSource source = new DOMSource(doc);
+ StreamResult result = new StreamResult(file);
+ transformer.transform(source, result);
+
+ // Output for testing
+ StreamResult consoleResult = new StreamResult(System.out);
+ transformer.transform(source, consoleResult);
+ System.out.println();
+
+ } catch (Exception e)
+ {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/0912bb65/src/test/java/org/apache/pirk/wideskies/standalone/StandaloneTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/pirk/wideskies/standalone/StandaloneTest.java b/src/test/java/org/apache/pirk/wideskies/standalone/StandaloneTest.java
new file mode 100644
index 0000000..06c2bff
--- /dev/null
+++ b/src/test/java/org/apache/pirk/wideskies/standalone/StandaloneTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pirk.wideskies.standalone;
+
+import java.util.ArrayList;
+
+import org.apache.pirk.schema.data.DataSchemaRegistry;
+import org.apache.pirk.schema.query.QuerySchemaRegistry;
+import org.apache.pirk.schema.query.filter.StopListFilter;
+import org.apache.pirk.test.utils.BaseTests;
+import org.apache.pirk.test.utils.Inputs;
+import org.apache.pirk.utils.SystemConfiguration;
+import org.json.simple.JSONObject;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Functional test suite for stand alone testing - non Spark applications
+ * <p>
+ * Tests low side module and basic encryption, decryption mechanisms
+ * <p>
+ * Using a fixed 8-bit data partition size (consistent with the currently codebase)
+ * <p>
+ * Runs with useExpLookupTable = false as generating the lookup table takes too long for normal in-memory builds
+ *
+ */
+public class StandaloneTest
+{
+ private static final Logger logger = LoggerFactory.getLogger(StandaloneTest.class);
+
+ private static final String STOPLIST_FILE = "testStopListFile";
+
+ private static String stopListFileProp = null;
+
+ @BeforeClass
+ public static void setup() throws Exception
+ {
+ // Reset the schema properties and registries
+ DataSchemaRegistry.clearRegistry();
+ QuerySchemaRegistry.clearRegistry();
+ SystemConfiguration.setProperty("data.schemas", "none");
+ SystemConfiguration.setProperty("query.schemas", "none");
+
+ // Create the stoplist file
+ stopListFileProp = SystemConfiguration.getProperty("pir.stopListFile");
+ SystemConfiguration.setProperty("pir.stopListFile", STOPLIST_FILE);
+ String newSLFile = Inputs.createPIRStopList(null, false);
+ SystemConfiguration.setProperty("pir.stopListFile", newSLFile);
+ logger.info("stopListFileProp = " + stopListFileProp + " new prop = " + SystemConfiguration.getProperty("pir.stopListFile"));
+
+ // Create data and query schemas
+ Inputs.createSchemaFiles(StopListFilter.class.getName());
+ }
+
+ @AfterClass
+ public static void teardown()
+ {
+ // Reset the schema properties and registries
+ DataSchemaRegistry.clearRegistry();
+ QuerySchemaRegistry.clearRegistry();
+ SystemConfiguration.setProperty("data.schemas", "none");
+ SystemConfiguration.setProperty("query.schemas", "none");
+ }
+
+ @Test
+ public void runTests() throws Exception
+ {
+ ArrayList<JSONObject> dataElements = Inputs.createJSONDataElements();
+ ArrayList<JSONObject> dataElementsRcode3 = Inputs.getRcode3JSONDataElements();
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+ // Run tests and use the embedded selector
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, 1, false);
+ BaseTests.testSRCIPQuery(dataElements, 2);
+ BaseTests.testDNSIPQuery(dataElements, 3); // numThreads % num elements to encrypt != 0
+ BaseTests.testDNSNXDOMAINQuery(dataElementsRcode3, 4); // numThreads % num elements to encrypt = 0
+
+ // Test embedded QuerySchema
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+ BaseTests.testDNSHostnameQuery(dataElements, 1, false);
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, 1, false);
+
+ SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false");
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, 1, false);
+ SystemConfiguration.setProperty("pir.embedQuerySchema", "false");
+
+ // Run tests without using the embedded selector
+ SystemConfiguration.setProperty("pirTest.embedSelector", "false");
+ BaseTests.testDNSHostnameQuery(dataElements, 1, false);
+ BaseTests.testSRCIPQuery(dataElements, 2);
+ BaseTests.testDNSIPQuery(dataElements, 3);
+ BaseTests.testDNSNXDOMAINQuery(dataElementsRcode3, 4);
+
+ // Run using a false positive
+ SystemConfiguration.setProperty("pirTest.embedSelector", "true");
+ BaseTests.testDNSHostnameQuery(dataElements, 1, true);
+
+ // Reset the stoplist file property
+ SystemConfiguration.setProperty("pir.stopListFile", stopListFileProp);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/0912bb65/src/test/java/test/general/ISO8601DateParserTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/test/general/ISO8601DateParserTest.java b/src/test/java/test/general/ISO8601DateParserTest.java
deleted file mode 100644
index 02391d4..0000000
--- a/src/test/java/test/general/ISO8601DateParserTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package test.general;
-
-import static org.junit.Assert.assertEquals;
-
-import java.text.ParseException;
-
-import org.apache.pirk.utils.ISO8601DateParser;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Class to test basic functionality of ISO8601DateParser class
- */
-public class ISO8601DateParserTest
-{
- private static final Logger logger = LoggerFactory.getLogger(ISO8601DateParserTest.class);
-
- @Test
- public void testDateParsing() throws ParseException
- {
- logger.info("Starting testDateParsing: ");
-
- String date = "2016-02-20T23:29:05.000Z";
- long longDate = Long.parseLong("1456010945000"); // date in UTC
-
- assertEquals(longDate, ISO8601DateParser.getLongDate(date));
- assertEquals(date, ISO8601DateParser.fromLongDate(longDate));
-
- logger.info("Successfully completed testDateParsing");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/0912bb65/src/test/java/test/general/KeyedHashTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/test/general/KeyedHashTest.java b/src/test/java/test/general/KeyedHashTest.java
deleted file mode 100644
index 6f69b38..0000000
--- a/src/test/java/test/general/KeyedHashTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package test.general;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.pirk.utils.KeyedHash;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Basic functional tests for KeyedHash
- *
- */
-public class KeyedHashTest
-{
- private static final Logger logger = LoggerFactory.getLogger(KeyedHashTest.class);
-
- @Test
- public void testKeyedHash()
- {
- logger.info("Starting testKeyedHash: ");
-
- int hash1 = KeyedHash.hash("someKey", 12, "someInput");
- logger.info("hash1 = " + hash1 + " hash1 = " + Integer.toString(hash1, 2));
-
- int hash2 = KeyedHash.hash("someKey", 32, "someInput");
- logger.info("hash2 = " + hash2 + " hash2 = " + Integer.toString(hash2, 2));
-
- int hash3 = KeyedHash.hash("someKey", 34, "someInput");
- logger.info("hash3 = " + hash3 + " hash3 = " + Integer.toString(hash3, 2));
-
- assertEquals(hash2, hash3);
- assertEquals(hash1, hash2 & 0xFFF);
-
- logger.info("Successfully completed testKeyedHash");
- }
-
- @Test
- public void testKeyedHashWithType()
- {
- testKeyedHashType("MD5");
- testKeyedHashType("SHA-1");
- testKeyedHashType("SHA-256");
- testKeyedHashType("FAKE-HASH-TYPE");
- }
-
- private void testKeyedHashType(String type)
- {
- logger.info("Starting testKeyedHashType with type: " + type);
-
- int hash1 = KeyedHash.hash("someKey", 12, "someInput", type);
- logger.info("hash1 = " + hash1 + " hash1 = " + Integer.toString(hash1, 2));
-
- int hash2 = KeyedHash.hash("someKey", 32, "someInput", type);
- logger.info("hash2 = " + hash2 + " hash2 = " + Integer.toString(hash2, 2));
-
- int hash3 = KeyedHash.hash("someKey", 34, "someInput", type);
- logger.info("hash3 = " + hash3 + " hash3 = " + Integer.toString(hash3, 2));
-
- assertEquals(hash2, hash3);
- assertEquals(hash1, hash2 & 0xFFF);
-
- logger.info("Successfully completed testKeyedHashType with type: " + type);
- }
-}