You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by ma...@apache.org on 2018/07/12 07:55:56 UTC
[incubator-openwhisk] branch master updated: Allow Elasticsearch
client to skip query result hits. (#3849)
This is an automated email from the ASF dual-hosted git repository.
markusthoemmes pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new bcba2d1 Allow Elasticsearch client to skip query result hits. (#3849)
bcba2d1 is described below
commit bcba2d1d2228e17c5bdb6e9d3cf666fd946b9216
Author: James Dubee <jw...@us.ibm.com>
AuthorDate: Thu Jul 12 03:55:53 2018 -0400
Allow Elasticsearch client to skip query result hits. (#3849)
---
.../logging/ElasticSearchRestClient.scala | 11 ++-----
.../logging/ElasticSearchLogStoreTests.scala | 3 +-
.../logging/ElasticSearchRestClientTests.scala | 36 +++++++++++++++-------
3 files changed, 30 insertions(+), 20 deletions(-)
diff --git a/common/scala/src/main/scala/whisk/core/containerpool/logging/ElasticSearchRestClient.scala b/common/scala/src/main/scala/whisk/core/containerpool/logging/ElasticSearchRestClient.scala
index 0290e96..9218958 100644
--- a/common/scala/src/main/scala/whisk/core/containerpool/logging/ElasticSearchRestClient.scala
+++ b/common/scala/src/main/scala/whisk/core/containerpool/logging/ElasticSearchRestClient.scala
@@ -57,7 +57,6 @@ case class EsQueryAggs(aggField: String, agg: EsAgg, field: String)
case class EsQueryRange(key: String, range: EsRange, value: String)
case class EsQueryBoolMatch(key: String, value: String)
case class EsQueryOrder(field: String, kind: EsOrder)
-case class EsQuerySize(size: Integer)
case class EsQueryAll() extends EsQueryMethod
case class EsQueryMust(matches: Vector[EsQueryBoolMatch], range: Vector[EsQueryRange] = Vector.empty)
extends EsQueryMethod
@@ -66,7 +65,8 @@ case class EsQueryTerm(key: String, value: String) extends EsQueryMethod
case class EsQueryString(queryString: String) extends EsQueryMethod
case class EsQuery(query: EsQueryMethod,
sort: Option[EsQueryOrder] = None,
- size: Option[EsQuerySize] = None,
+ size: Option[Int] = None,
+ from: Int = 0,
aggs: Option[EsQueryAggs] = None)
// Schema of ES query results
@@ -121,11 +121,6 @@ object ElasticSearchJsonProtocol extends DefaultJsonProtocol {
JsArray(JsObject(query.field -> JsObject("order" -> query.kind.toString.toJson)))
}
- implicit object EsQuerySizeJsonFormat extends RootJsonFormat[EsQuerySize] {
- def read(query: JsValue) = ???
- def write(query: EsQuerySize) = JsNumber(query.size)
- }
-
implicit object EsQueryAggsJsonFormat extends RootJsonFormat[EsQueryAggs] {
def read(query: JsValue) = ???
def write(query: EsQueryAggs) =
@@ -148,7 +143,7 @@ object ElasticSearchJsonProtocol extends DefaultJsonProtocol {
}
}
- implicit val esQueryFormat = jsonFormat4(EsQuery.apply)
+ implicit val esQueryFormat = jsonFormat5(EsQuery.apply)
implicit val esSearchHitFormat = jsonFormat(EsSearchHit.apply _, "_source")
implicit val esSearchHitsFormat = jsonFormat2(EsSearchHits.apply)
implicit val esSearchResultFormat = jsonFormat1(EsSearchResult.apply)
diff --git a/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchLogStoreTests.scala b/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchLogStoreTests.scala
index bd3e0a0..91db374 100644
--- a/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchLogStoreTests.scala
+++ b/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchLogStoreTests.scala
@@ -79,7 +79,8 @@ class ElasticSearchLogStoreTests
"query" -> JsObject(
"query_string" -> JsObject("query" -> JsString(
s"_type: ${defaultConfig.logSchema.userLogs} AND ${defaultConfig.logSchema.activationId}: $activationId"))),
- "sort" -> JsArray(JsObject(defaultConfig.logSchema.time -> JsObject("order" -> JsString("asc"))))).compactPrint
+ "sort" -> JsArray(JsObject(defaultConfig.logSchema.time -> JsObject("order" -> JsString("asc")))),
+ "from" -> JsNumber(0)).compactPrint
private val defaultHttpRequest = HttpRequest(
POST,
Uri(s"/whisk_user_logs/_search"),
diff --git a/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchRestClientTests.scala b/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchRestClientTests.scala
index 040b0fc..2a43b79 100644
--- a/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchRestClientTests.scala
+++ b/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchRestClientTests.scala
@@ -87,7 +87,8 @@ class ElasticSearchRestClientTests
"must" ->
JsArray(
JsObject("match" -> JsObject("someKey1" -> JsString("someValue1"))),
- JsObject("match" -> JsObject("someKey2" -> JsString("someValue2")))))))
+ JsObject("match" -> JsObject("someKey2" -> JsString("someValue2")))))),
+ "from" -> 0.toJson)
// Test must with ranges
Seq((EsRangeGte, "gte"), (EsRangeGt, "gt"), (EsRangeLte, "lte"), (EsRangeLt, "lt")).foreach {
@@ -109,7 +110,8 @@ class ElasticSearchRestClientTests
"filter" ->
JsArray(
JsObject("range" -> JsObject("someKey1" -> JsObject(rangeValue -> "someValue1".toJson))),
- JsObject("range" -> JsObject("someKey2" -> JsObject(rangeValue -> "someValue2".toJson)))))))
+ JsObject("range" -> JsObject("someKey2" -> JsObject(rangeValue -> "someValue2".toJson)))))),
+ "from" -> 0.toJson)
}
}
@@ -120,7 +122,8 @@ class ElasticSearchRestClientTests
EsQuery(EsQueryAll(), aggs = Some(queryAgg)).toJson shouldBe JsObject(
"query" -> JsObject("match_all" -> JsObject.empty),
- "aggs" -> JsObject("someAgg" -> JsObject(aggValue -> JsObject("field" -> "someField".toJson))))
+ "aggs" -> JsObject("someAgg" -> JsObject(aggValue -> JsObject("field" -> "someField".toJson))),
+ "from" -> 0.toJson)
}
}
@@ -128,7 +131,8 @@ class ElasticSearchRestClientTests
val queryMatch = EsQueryMatch("someField", "someValue")
EsQuery(queryMatch).toJson shouldBe JsObject(
- "query" -> JsObject("match" -> JsObject("someField" -> JsObject("query" -> "someValue".toJson))))
+ "query" -> JsObject("match" -> JsObject("someField" -> JsObject("query" -> "someValue".toJson))),
+ "from" -> 0.toJson)
// Test match with types
Seq((EsMatchPhrase, "phrase"), (EsMatchPhrasePrefix, "phrase_prefix")).foreach {
@@ -137,21 +141,25 @@ class ElasticSearchRestClientTests
EsQuery(queryMatch).toJson shouldBe JsObject(
"query" -> JsObject(
- "match" -> JsObject("someField" -> JsObject("query" -> "someValue".toJson, "type" -> typeValue.toJson))))
+ "match" -> JsObject("someField" -> JsObject("query" -> "someValue".toJson, "type" -> typeValue.toJson))),
+ "from" -> 0.toJson)
}
}
it should "construct a query with term" in {
val queryTerm = EsQueryTerm("user", "someUser")
- EsQuery(queryTerm).toJson shouldBe JsObject("query" -> JsObject("term" -> JsObject("user" -> JsString("someUser"))))
+ EsQuery(queryTerm).toJson shouldBe JsObject(
+ "query" -> JsObject("term" -> JsObject("user" -> JsString("someUser"))),
+ "from" -> 0.toJson)
}
it should "construct a query with query string" in {
val queryString = EsQueryString("_type: someType")
EsQuery(queryString).toJson shouldBe JsObject(
- "query" -> JsObject("query_string" -> JsObject("query" -> JsString("_type: someType"))))
+ "query" -> JsObject("query_string" -> JsObject("query" -> JsString("_type: someType"))),
+ "from" -> 0.toJson)
}
it should "construct a query with order" in {
@@ -161,16 +169,22 @@ class ElasticSearchRestClientTests
EsQuery(EsQueryAll(), Some(queryOrder)).toJson shouldBe JsObject(
"query" -> JsObject("match_all" -> JsObject.empty),
- "sort" -> JsArray(JsObject("someField" -> JsObject("order" -> orderValue.toJson))))
+ "sort" -> JsArray(JsObject("someField" -> JsObject("order" -> orderValue.toJson))),
+ "from" -> 0.toJson)
}
}
it should "construct query with size" in {
- val querySize = EsQuerySize(1)
+ EsQuery(EsQueryAll(), size = Some(1)).toJson shouldBe JsObject(
+ "query" -> JsObject("match_all" -> JsObject.empty),
+ "size" -> 1.toJson,
+ "from" -> 0.toJson)
+ }
- EsQuery(EsQueryAll(), size = Some(querySize)).toJson shouldBe JsObject(
+ it should "construct query with from" in {
+ EsQuery(EsQueryAll(), from = 1).toJson shouldBe JsObject(
"query" -> JsObject("match_all" -> JsObject.empty),
- "size" -> JsNumber(1))
+ "from" -> 1.toJson)
}
it should "error when search response does not match expected type" in {