You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by jm...@apache.org on 2021/07/30 17:40:27 UTC
[datasketches-server] branch test_coverage updated (4e5b593 ->
f5da446)
This is an automated email from the ASF dual-hosted git repository.
jmalkin pushed a change to branch test_coverage
in repository https://gitbox.apache.org/repos/asf/datasketches-server.git.
from 4e5b593 add reset tests, refactor testing base class, to better reflect null return value
new d3b1d33 document test base class a bit more, test status calls
new f5da446 Remove some hard-coded names from reset tests, add more serialization tests, update get/post methods to accept JsonElement to support arrays rather than single items
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../datasketches/server/ResetHandlerTest.java | 19 +-
.../server/SerializationHandlerTest.java | 209 +++++++++++++++++++++
.../apache/datasketches/server/ServerTestBase.java | 24 ++-
.../datasketches/server/StatusHandlerTest.java | 61 ++++++
4 files changed, 302 insertions(+), 11 deletions(-)
create mode 100644 src/test/java/org/apache/datasketches/server/SerializationHandlerTest.java
create mode 100644 src/test/java/org/apache/datasketches/server/StatusHandlerTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[datasketches-server] 01/02: document test base class a bit more,
test status calls
Posted by jm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jmalkin pushed a commit to branch test_coverage
in repository https://gitbox.apache.org/repos/asf/datasketches-server.git
commit d3b1d3312710028528405e9f48b06899f36fd33d
Author: Jon Malkin <jm...@users.noreply.github.com>
AuthorDate: Thu Jul 29 11:45:17 2021 -0700
document test base class a bit more, test status calls
---
.../apache/datasketches/server/ServerTestBase.java | 20 +++++++
.../datasketches/server/StatusHandlerTest.java | 61 ++++++++++++++++++++++
2 files changed, 81 insertions(+)
diff --git a/src/test/java/org/apache/datasketches/server/ServerTestBase.java b/src/test/java/org/apache/datasketches/server/ServerTestBase.java
index a6df4f2..30865c3 100644
--- a/src/test/java/org/apache/datasketches/server/ServerTestBase.java
+++ b/src/test/java/org/apache/datasketches/server/ServerTestBase.java
@@ -77,6 +77,14 @@ public class ServerTestBase {
}
}
+ /**
+ * Sends a POST to the server at the target path, using the provided data. Returns the HTTP status code
+ * as well as any response in the response object (after clearing out anything that already existed).
+ * @param path Request path appended to the server URI
+ * @param data Request data as a JsonObject
+ * @param response Response data as a JsonObject -- existing data will be removed
+ * @return The HTTP status code from the server
+ */
int postData(@NonNull final String path,
@NonNull final JsonObject data,
@NonNull final JsonObject response) {
@@ -132,6 +140,18 @@ public class ServerTestBase {
return status;
}
+ /**
+ * <p>Sends a GET to the server at the target path, using the provided data. Returns the HTTP status code
+ * as well as any response in the response object (after clearing out anything that already existed).</p>
+ *
+ * <p>Unlike POST, this call may be subject to HTTP header length limits since the data is encoded as
+ * the request querystrong.</p>
+ * @param path Request path appended to the server URI
+ * @param data Request data as a JsonObject
+ * @param response Response data as a JsonObject -- existing data will be removed
+ * @return The HTTP status code from the server
+ */
+
int getData(@NonNull final String path,
@NonNull final JsonObject data,
@NonNull final JsonObject response) {
diff --git a/src/test/java/org/apache/datasketches/server/StatusHandlerTest.java b/src/test/java/org/apache/datasketches/server/StatusHandlerTest.java
new file mode 100644
index 0000000..c91119d
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/server/StatusHandlerTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.server;
+
+import static org.apache.datasketches.server.SketchConstants.RESPONSE_SKETCH_COUNT_FIELD;
+import static org.apache.datasketches.server.SketchConstants.STATUS_PATH;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+import org.eclipse.jetty.http.HttpStatus;
+import org.testng.annotations.Test;
+
+
+import com.google.gson.JsonObject;
+
+public class StatusHandlerTest extends ServerTestBase {
+ @Test
+ public void callStatus() {
+ // really simple call -- the parameters are ignored
+ final JsonObject response = new JsonObject();
+ JsonObject request = new JsonObject();
+
+ // GET will easily work since the header size isn't an issue with no data
+ assertEquals(getData(STATUS_PATH, request, response), HttpStatus.OK_200);
+ JsonObject data = response.get(RESPONSE_FIELD).getAsJsonObject();
+ assertTrue(data.has(RESPONSE_SKETCH_COUNT_FIELD));
+ assertEquals(data.get(RESPONSE_SKETCH_COUNT_FIELD).getAsInt(), 20);
+
+ // POST the same query
+ request.addProperty("notAName", "theta0");
+ assertEquals(postData(STATUS_PATH, request, response), HttpStatus.OK_200);
+ data = response.get(RESPONSE_FIELD).getAsJsonObject();
+ assertTrue(data.has(RESPONSE_SKETCH_COUNT_FIELD));
+ assertEquals(data.get(RESPONSE_SKETCH_COUNT_FIELD).getAsInt(), 20);
+
+ // send in a tiny bit of data that should be ignored
+ request = new JsonObject();
+ request.add("name", new JsonObject());
+ assertEquals(postData(STATUS_PATH, request, response), HttpStatus.OK_200);
+ data = response.get(RESPONSE_FIELD).getAsJsonObject();
+ assertTrue(data.has(RESPONSE_SKETCH_COUNT_FIELD));
+ assertEquals(data.get(RESPONSE_SKETCH_COUNT_FIELD).getAsInt(), 20);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[datasketches-server] 02/02: Remove some hard-coded names from
reset tests, add more serialization tests,
update get/post methods to accept JsonElement to support arrays rather than
single items
Posted by jm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jmalkin pushed a commit to branch test_coverage
in repository https://gitbox.apache.org/repos/asf/datasketches-server.git
commit f5da4465ae0865395a4ca437d499d9fd1cb198d5
Author: Jon Malkin <jm...@users.noreply.github.com>
AuthorDate: Fri Jul 30 10:40:19 2021 -0700
Remove some hard-coded names from reset tests, add more serialization tests, update get/post methods to accept JsonElement to support arrays rather than single items
---
.../datasketches/server/ResetHandlerTest.java | 19 +-
.../server/SerializationHandlerTest.java | 209 +++++++++++++++++++++
.../apache/datasketches/server/ServerTestBase.java | 8 +-
3 files changed, 223 insertions(+), 13 deletions(-)
diff --git a/src/test/java/org/apache/datasketches/server/ResetHandlerTest.java b/src/test/java/org/apache/datasketches/server/ResetHandlerTest.java
index f2e2e4d..0ead4d2 100644
--- a/src/test/java/org/apache/datasketches/server/ResetHandlerTest.java
+++ b/src/test/java/org/apache/datasketches/server/ResetHandlerTest.java
@@ -19,6 +19,7 @@
package org.apache.datasketches.server;
+import static org.apache.datasketches.server.SketchConstants.QUERY_NAME_FIELD;
import static org.apache.datasketches.server.SketchConstants.RESET_PATH;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
@@ -51,12 +52,12 @@ public class ResetHandlerTest extends ServerTestBase {
// valid name, empty value
request = new JsonObject();
- request.add("name", new JsonObject());
+ request.add(QUERY_NAME_FIELD, new JsonObject());
assertEquals(postData(RESET_PATH, request, response), HttpStatus.UNPROCESSABLE_ENTITY_422);
// valid name, invalid sketch name
request = new JsonObject();
- request.addProperty("name", "sketchDoesNotExist");
+ request.addProperty(QUERY_NAME_FIELD, "sketchDoesNotExist");
assertEquals(postData(RESET_PATH, request, response), HttpStatus.UNPROCESSABLE_ENTITY_422);
}
@@ -72,7 +73,7 @@ public class ResetHandlerTest extends ServerTestBase {
assertFalse(sk.isEmpty());
// reset, then check sketch is again empty
- request.addProperty("name", sketchName);
+ request.addProperty(QUERY_NAME_FIELD, sketchName);
assertEquals(getData(RESET_PATH, request, response), HttpStatus.OK_200);
assertEquals(response.size(), 0);
@@ -95,7 +96,7 @@ public class ResetHandlerTest extends ServerTestBase {
assertFalse(sk.isEmpty());
// reset, then check sketch is again empty
- request.addProperty("name", sketchName);
+ request.addProperty(QUERY_NAME_FIELD, sketchName);
assertEquals(getData(RESET_PATH, request, response), HttpStatus.OK_200);
assertEquals(response.size(), 0);
@@ -117,7 +118,7 @@ public class ResetHandlerTest extends ServerTestBase {
assertFalse(sk.isEmpty());
// reset, then check sketch is again empty
- request.addProperty("name", sketchName);
+ request.addProperty(QUERY_NAME_FIELD, sketchName);
assertEquals(getData(RESET_PATH, request, response), HttpStatus.OK_200);
assertEquals(response.size(), 0);
@@ -138,7 +139,7 @@ public class ResetHandlerTest extends ServerTestBase {
assertFalse(sk.isEmpty());
// reset, then check sketch is again empty
- request.addProperty("name", sketchName);
+ request.addProperty(QUERY_NAME_FIELD, sketchName);
assertEquals(getData(RESET_PATH, request, response), HttpStatus.OK_200);
assertEquals(response.size(), 0);
@@ -159,7 +160,7 @@ public class ResetHandlerTest extends ServerTestBase {
assertFalse(sk.getResult().isEmpty());
// reset, then check sketch is again empty
- request.addProperty("name", sketchName);
+ request.addProperty(QUERY_NAME_FIELD, sketchName);
assertEquals(getData(RESET_PATH, request, response), HttpStatus.OK_200);
assertEquals(response.size(), 0);
@@ -181,7 +182,7 @@ public class ResetHandlerTest extends ServerTestBase {
assertTrue(sk.getNumSamples() > 0);
// reset, then check sketch is again empty
- request.addProperty("name", sketchName);
+ request.addProperty(QUERY_NAME_FIELD, sketchName);
assertEquals(getData(RESET_PATH, request, response), HttpStatus.OK_200);
assertEquals(response.size(), 0);
@@ -203,7 +204,7 @@ public class ResetHandlerTest extends ServerTestBase {
assertTrue(sk.getNumSamples() > 0);
// reset, then check sketch is again empty
- request.addProperty("name", sketchName);
+ request.addProperty(QUERY_NAME_FIELD, sketchName);
assertEquals(getData(RESET_PATH, request, response), HttpStatus.OK_200);
assertEquals(response.size(), 0);
diff --git a/src/test/java/org/apache/datasketches/server/SerializationHandlerTest.java b/src/test/java/org/apache/datasketches/server/SerializationHandlerTest.java
new file mode 100644
index 0000000..676b211
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/server/SerializationHandlerTest.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.server;
+
+import static org.apache.datasketches.server.SketchConstants.QUERY_NAME_FIELD;
+import static org.apache.datasketches.server.SketchConstants.QUERY_SKETCH_FIELD;
+import static org.apache.datasketches.server.SketchConstants.SERIALIZE_PATH;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import java.util.Base64;
+
+import org.apache.datasketches.ArrayOfStringsSerDe;
+import org.apache.datasketches.cpc.CpcSketch;
+import org.apache.datasketches.frequencies.ItemsSketch;
+import org.apache.datasketches.hll.HllSketch;
+import org.apache.datasketches.kll.KllFloatsSketch;
+import org.apache.datasketches.sampling.ReservoirItemsSketch;
+import org.apache.datasketches.sampling.VarOptItemsSketch;
+import org.apache.datasketches.theta.Union;
+import org.eclipse.jetty.http.HttpStatus;
+import org.testng.annotations.Test;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+public class SerializationHandlerTest extends ServerTestBase {
+ void checkSerializedResult(final byte[] skBytes, final String tgtSketchName) {
+ final JsonObject response = new JsonObject();
+ final JsonObject request = new JsonObject();
+
+ request.addProperty(QUERY_NAME_FIELD, tgtSketchName);
+ assertEquals(getData(SERIALIZE_PATH, request, response), HttpStatus.OK_200);
+ final JsonObject result = response.get(RESPONSE_FIELD).getAsJsonObject();
+
+ // expecting a single entry in the response, so decode and compare against serialized sketch
+ assertEquals(result.get(QUERY_NAME_FIELD).getAsString(), tgtSketchName);
+ final String b64Data = result.get(QUERY_SKETCH_FIELD).getAsString();
+ assertEquals(Base64.getUrlDecoder().decode(b64Data), skBytes);
+ }
+
+ @Test
+ public void emptySerialize() {
+ final JsonObject response = new JsonObject();
+ JsonObject request = new JsonObject();
+
+ // completely empty request cannot be handled
+ assertEquals(postData(SERIALIZE_PATH, request, response), HttpStatus.UNPROCESSABLE_ENTITY_422);
+
+ // invalid name field with real sketch name
+ request.addProperty("notAName", "theta0");
+ assertEquals(postData(SERIALIZE_PATH, request, response), HttpStatus.UNPROCESSABLE_ENTITY_422);
+
+ // valid name field, empty value
+ request = new JsonObject();
+ request.add(QUERY_NAME_FIELD, new JsonObject());
+ assertEquals(postData(SERIALIZE_PATH, request, response), HttpStatus.UNPROCESSABLE_ENTITY_422);
+
+ // valid name, invalid sketch name
+ request = new JsonObject();
+ request.addProperty(QUERY_NAME_FIELD, "sketchDoesNotExist");
+ assertEquals(postData(SERIALIZE_PATH, request, response), HttpStatus.UNPROCESSABLE_ENTITY_422);
+ }
+
+ @Test
+ public void cpcSerialize() {
+ final String sketchName = "cpcOfLongs";
+
+ // add data directly to sketch
+ final CpcSketch sk = (CpcSketch) server_.getSketch(sketchName).sketch_;
+ for (int i = 0; i < 5; ++i) { sk.update(i); }
+ assertFalse(sk.isEmpty());
+ final byte[] skBytes = sk.toByteArray();
+
+ checkSerializedResult(skBytes, sketchName);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void fiSerialize() {
+ final String sketchName = "topItems";
+
+ final ItemsSketch<String> sk = (ItemsSketch<String>) server_.getSketch(sketchName).sketch_;
+ sk.update("itemA", 3);
+ sk.update("itemB", 1);
+ final byte[] skBytes = sk.toByteArray(new ArrayOfStringsSerDe());
+
+ checkSerializedResult(skBytes, sketchName);
+ }
+
+ @Test
+ public void hllSerialization() {
+ final String sketchName1 = "hll1";
+ final String sketchName2 = "hll2";
+
+ // one one data, one empty, sent in as an array
+ final HllSketch sk1 = (HllSketch) server_.getSketch(sketchName1).sketch_;
+ sk1.update("itemA");
+ sk1.update("itemB");
+ final byte[] sk1Bytes = sk1.toCompactByteArray();
+ final HllSketch sk2 = (HllSketch) server_.getSketch(sketchName2).sketch_;
+ final byte[] sk2Bytes = sk2.toCompactByteArray();
+
+ // create input as an array
+ final JsonArray request = new JsonArray();
+ final JsonObject obj1 = new JsonObject();
+ obj1.addProperty(QUERY_NAME_FIELD, sketchName1);
+ request.add(obj1);
+ final JsonObject obj2 = new JsonObject();
+ obj2.addProperty(QUERY_NAME_FIELD, sketchName2);
+ request.add(obj2);
+
+ final JsonObject response = new JsonObject();
+ assertEquals(getData(SERIALIZE_PATH, request, response), HttpStatus.OK_200);
+ final JsonArray responseData = response.getAsJsonArray(RESPONSE_FIELD);
+ assertEquals(responseData.size(), 2);
+
+ boolean foundSk1 = false;
+ boolean foundSk2 = false;
+ String b64Data;
+ for (final JsonElement e : responseData) {
+ final JsonObject o = e.getAsJsonObject();
+ switch (o.get(QUERY_NAME_FIELD).getAsString()) {
+ case sketchName1:
+ b64Data = o.get(QUERY_SKETCH_FIELD).getAsString();
+ assertEquals(Base64.getUrlDecoder().decode(b64Data), sk1Bytes);
+ foundSk1 = true;
+ break;
+ case sketchName2:
+ b64Data = o.get(QUERY_SKETCH_FIELD).getAsString();
+ assertEquals(Base64.getUrlDecoder().decode(b64Data), sk2Bytes);
+ foundSk2 = true;
+ break;
+ default:
+ fail();
+ }
+ }
+ assertTrue(foundSk1 && foundSk2);
+ }
+
+ @Test
+ public void kllSerialization() {
+ final String sketchName = "duration";
+
+ final KllFloatsSketch sk = (KllFloatsSketch) server_.getSketch(sketchName).sketch_;
+ for (int i = 0; i < 2942; ++i) { sk.update(i * 128.0f); }
+ final byte[] skBytes = sk.toByteArray();
+
+ checkSerializedResult(skBytes, sketchName);
+ }
+
+ @Test
+ public void thetaSerialization() {
+ final String sketchName = "theta1";
+
+ final Union sk = (Union) server_.getSketch(sketchName).sketch_;
+ sk.update("item");
+ final byte[] skBytes = sk.getResult().toByteArray();
+
+ checkSerializedResult(skBytes, sketchName);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void reservoirReset() {
+ final String sketchName = "rs";
+
+ final ReservoirItemsSketch<String> sk = (ReservoirItemsSketch<String>) server_.getSketch(sketchName).sketch_;
+ for (int i = 0; i < 500; ++i) { sk.update("item" + i); }
+ final byte[] skBytes = sk.toByteArray(new ArrayOfStringsSerDe());
+
+ checkSerializedResult(skBytes, sketchName);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void varOptReset() {
+ final String sketchName = "vo";
+
+ // add some light items and a few really heavy things
+ VarOptItemsSketch<String> sk = (VarOptItemsSketch<String>) server_.getSketch(sketchName).sketch_;
+ for (int i = 0; i < 250; ++i) { sk.update("item" + i, 1.0); }
+ sk.update("heavyItem1", 1e6);
+ sk.update("heavyItem2", 1.1e6);
+ final byte[] skBytes = sk.toByteArray(new ArrayOfStringsSerDe());
+
+ checkSerializedResult(skBytes, sketchName);
+ }
+}
diff --git a/src/test/java/org/apache/datasketches/server/ServerTestBase.java b/src/test/java/org/apache/datasketches/server/ServerTestBase.java
index 30865c3..e6e41bf 100644
--- a/src/test/java/org/apache/datasketches/server/ServerTestBase.java
+++ b/src/test/java/org/apache/datasketches/server/ServerTestBase.java
@@ -81,12 +81,12 @@ public class ServerTestBase {
* Sends a POST to the server at the target path, using the provided data. Returns the HTTP status code
* as well as any response in the response object (after clearing out anything that already existed).
* @param path Request path appended to the server URI
- * @param data Request data as a JsonObject
+ * @param data Request data as a JsonElement
* @param response Response data as a JsonObject -- existing data will be removed
* @return The HTTP status code from the server
*/
int postData(@NonNull final String path,
- @NonNull final JsonObject data,
+ @NonNull final JsonElement data,
@NonNull final JsonObject response) {
HttpURLConnection http = null;
int status = -1;
@@ -147,13 +147,13 @@ public class ServerTestBase {
* <p>Unlike POST, this call may be subject to HTTP header length limits since the data is encoded as
* the request querystrong.</p>
* @param path Request path appended to the server URI
- * @param data Request data as a JsonObject
+ * @param data Request data as a JsonElement
* @param response Response data as a JsonObject -- existing data will be removed
* @return The HTTP status code from the server
*/
int getData(@NonNull final String path,
- @NonNull final JsonObject data,
+ @NonNull final JsonElement data,
@NonNull final JsonObject response) {
HttpURLConnection http = null;
int status = -1;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org