You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zipkin.apache.org by ad...@apache.org on 2019/05/03 11:08:54 UTC
[incubator-zipkin] branch master updated: Supports Elasticsearch
7.x (#2398)
This is an automated email from the ASF dual-hosted git repository.
adriancole pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-zipkin.git
The following commit(s) were added to refs/heads/master by this push:
new 762a795 Supports Elasticsearch 7.x (#2398)
762a795 is described below
commit 762a79539ac3ed945038ee759cfec3a28971105c
Author: Adrian Cole <ad...@users.noreply.github.com>
AuthorDate: Fri May 3 19:08:49 2019 +0800
Supports Elasticsearch 7.x (#2398)
Fixes #2219
---
...nElasticsearchStorageAutoConfigurationTest.java | 39 +-----
zipkin-storage/elasticsearch/README.md | 15 ++-
.../elasticsearch/ElasticsearchSpanConsumer.java | 13 +-
.../elasticsearch/ElasticsearchStorage.java | 72 +++++-----
.../zipkin2/elasticsearch/EnsureIndexTemplate.java | 18 ++-
.../java/zipkin2/elasticsearch/IndexTemplates.java | 9 ++
.../elasticsearch/VersionSpecificTemplates.java | 65 +++++----
.../elasticsearch/internal/HttpBulkIndexer.java | 56 ++------
.../elasticsearch/internal/IndexNameFormatter.java | 49 ++++++-
.../ElasticsearchSpanConsumerTest.java | 1 -
.../elasticsearch/ElasticsearchStorageTest.java | 4 +-
.../zipkin2/elasticsearch/InternalForTests.java | 4 +-
.../VersionSpecificTemplatesTest.java | 49 ++++++-
.../integration/ElasticsearchStorageRule.java | 2 +-
.../integration/ITElasticsearchStorageV7.java | 145 +++++++++++++++++++++
.../internal/IndexNameFormatterTest.java | 88 ++++++-------
16 files changed, 411 insertions(+), 218 deletions(-)
diff --git a/zipkin-server/src/test/java/zipkin2/elasticsearch/ZipkinElasticsearchStorageAutoConfigurationTest.java b/zipkin-server/src/test/java/zipkin2/elasticsearch/ZipkinElasticsearchStorageAutoConfigurationTest.java
index 3f1d7ec..d9cfa8e 100644
--- a/zipkin-server/src/test/java/zipkin2/elasticsearch/ZipkinElasticsearchStorageAutoConfigurationTest.java
+++ b/zipkin-server/src/test/java/zipkin2/elasticsearch/ZipkinElasticsearchStorageAutoConfigurationTest.java
@@ -38,18 +38,15 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Rule public ExpectedException thrown = ExpectedException.none();
- AnnotationConfigApplicationContext context;
+ final AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext();
@After
public void close() {
- if (context != null) {
- context.close();
- }
+ context.close();
}
@Test
public void doesntProvideStorageComponent_whenStorageTypeNotElasticsearch() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of("zipkin.storage.type:cassandra").applyTo(context);
Access.registerElasticsearchHttp(context);
context.refresh();
@@ -60,7 +57,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void providesStorageComponent_whenStorageTypeElasticsearchAndHostsAreUrls() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200")
@@ -73,7 +69,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void canOverridesProperty_hostsWithList() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200,http://host2:9200")
@@ -87,7 +82,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void configuresPipeline() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200",
@@ -101,7 +95,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void configuresMaxRequests() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200",
@@ -116,7 +109,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
/** This helps ensure old setups don't break (provided they have http port 9200 open) */
@Test
public void coersesPort9300To9200() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:host1:9300")
@@ -129,7 +121,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void httpPrefixOptional() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:host1:9200")
@@ -142,7 +133,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void defaultsToPort9200() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:host1")
@@ -175,7 +165,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
/** Ensures we can wire up network interceptors, such as for logging or authentication */
@Test
public void usesInterceptorsQualifiedWith_zipkinElasticsearchHttp() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:host1:9200")
@@ -190,7 +179,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void timeout_defaultsTo10Seconds() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:host1:9200")
@@ -206,7 +194,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void timeout_override() {
- context = new AnnotationConfigApplicationContext();
int timeout = 30_000;
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
@@ -224,7 +211,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void strictTraceId_defaultsToTrue() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200")
@@ -236,7 +222,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void strictTraceId_canSetToFalse() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200",
@@ -250,7 +235,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void dailyIndexFormat() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200")
@@ -259,12 +243,11 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
context.refresh();
assertThat(es().indexNameFormatter().formatTypeAndTimestamp("span", 0))
- .isEqualTo("zipkin:span-1970-01-01");
+ .isEqualTo("zipkin*span-1970-01-01");
}
@Test
public void dailyIndexFormat_overridingPrefix() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200",
@@ -274,12 +257,11 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
context.refresh();
assertThat(es().indexNameFormatter().formatTypeAndTimestamp("span", 0))
- .isEqualTo("zipkin_prod:span-1970-01-01");
+ .isEqualTo("zipkin_prod*span-1970-01-01");
}
@Test
public void dailyIndexFormat_overridingDateSeparator() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200",
@@ -289,12 +271,11 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
context.refresh();
assertThat(es().indexNameFormatter().formatTypeAndTimestamp("span", 0))
- .isEqualTo("zipkin:span-1970.01.01");
+ .isEqualTo("zipkin*span-1970.01.01");
}
@Test
public void dailyIndexFormat_overridingDateSeparator_empty() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200",
@@ -304,12 +285,11 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
context.refresh();
assertThat(es().indexNameFormatter().formatTypeAndTimestamp("span", 0))
- .isEqualTo("zipkin:span-19700101");
+ .isEqualTo("zipkin*span-19700101");
}
@Test
public void dailyIndexFormat_overridingDateSeparator_invalidToBeMultiChar() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200",
@@ -323,7 +303,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void namesLookbackAssignedFromQueryLookback() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200",
@@ -337,7 +316,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void doesntProvideBasicAuthInterceptor_whenBasicAuthUserNameandPasswordNotConfigured() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200")
@@ -351,7 +329,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void providesBasicAuthInterceptor_whenBasicAuthUserNameAndPasswordConfigured() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.elasticsearch.hosts:http://host1:9200",
@@ -368,7 +345,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void searchEnabled_false() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.search-enabled:false")
@@ -381,7 +357,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void autocompleteKeys_list() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.autocomplete-keys:environment")
@@ -395,7 +370,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void autocompleteTtl() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.autocomplete-ttl:60000")
@@ -409,7 +383,6 @@ public class ZipkinElasticsearchStorageAutoConfigurationTest {
@Test
public void autocompleteCardinality() {
- context = new AnnotationConfigApplicationContext();
TestPropertyValues.of(
"zipkin.storage.type:elasticsearch",
"zipkin.storage.autocomplete-cardinality:5000")
diff --git a/zipkin-storage/elasticsearch/README.md b/zipkin-storage/elasticsearch/README.md
index f3bcc0a..a3ee202 100644
--- a/zipkin-storage/elasticsearch/README.md
+++ b/zipkin-storage/elasticsearch/README.md
@@ -3,7 +3,7 @@
This is is a plugin to the Elasticsearch storage component, which uses
HTTP by way of [OkHttp 3](https://github.com/square/okttp) and
[Moshi](https://github.com/square/moshi). This currently supports 2.x,
-5.x and 6.x version families.
+5.x, 6.x and 7.x version families.
## Multiple hosts
Most users will supply a DNS name that's mapped to multiple A or AAAA
@@ -33,7 +33,8 @@ spans. This is mapped to the Elasticsearch date type, so can be used to any date
## Indexes
Spans are stored into daily indices, for example spans with a timestamp
-falling on 2016/03/19 will be stored in the index named 'zipkin:span-2016-03-19'.
+falling on 2016/03/19 will be stored in the index named 'zipkin:span-2016-03-19'
+or 'zipkin-span-2016-03-19' if using Elasticsearch version 7 or higher.
There is no support for TTL through this SpanStore. It is recommended
instead to use [Elastic Curator](https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about.html)
to remove indices older than the point you are interested in.
@@ -45,9 +46,9 @@ the date separator from '-' to something else.
`ElasticsearchStorage.Builder.index` and `ElasticsearchStorage.Builder.dateSeparator`
control the daily index format.
-For example, spans with a timestamp falling on 2016/03/19 end up in the
-index 'zipkin:span-2016-03-19'. When the date separator is '.', the index
-would be 'zipkin:span-2016.03.19'.
+For example, using Elasticsearch 7+, spans with a timestamp falling on
+2016/03/19 end up in the index 'zipkin-span-2016-03-19'. When the date
+separator is '.', the index would be 'zipkin-span-2016.03.19'.
### String Mapping
The Zipkin api implies aggregation and exact match (keyword) on string
@@ -63,7 +64,7 @@ The values in `q` are limited to 256 characters and searched as keywords.
You can check these manually like so:
```bash
-$ curl -s localhost:9200/zipkin:span-2017-08-11/_search?q=_q:error=500
+$ curl -s 'localhost:9200/zipkin*span-2017-08-11/_search?q=_q:error=500'
```
The reason for special casing is around dotted name constraints. Tags
@@ -103,7 +104,7 @@ your indexes:
```bash
# the output below shows which tokens will match on the trace id supplied.
-$ curl -s localhost:9200/zipkin:span-2017-08-22/_analyze -d '{
+$ curl -s 'localhost:9200/zipkin*span-2017-08-22/_analyze' -d '{
"text": "48485a3953bb61246b221d5bc9e6496c",
"analyzer": "traceId_analyzer"
}'|jq '.tokens|.[]|.token'
diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanConsumer.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanConsumer.java
index 84993db..e793394 100644
--- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanConsumer.java
+++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchSpanConsumer.java
@@ -35,6 +35,7 @@ import zipkin2.codec.SpanBytesEncoder;
import zipkin2.elasticsearch.internal.HttpBulkIndexer;
import zipkin2.elasticsearch.internal.IndexNameFormatter;
import zipkin2.internal.DelayLimiter;
+import zipkin2.internal.Nullable;
import zipkin2.storage.SpanConsumer;
import static zipkin2.elasticsearch.ElasticsearchAutocompleteTags.AUTOCOMPLETE;
@@ -50,6 +51,7 @@ class ElasticsearchSpanConsumer implements SpanConsumer { // not final for testi
final ElasticsearchStorage es;
final Set<String> autocompleteKeys;
final IndexNameFormatter indexNameFormatter;
+ final char indexTypeDelimiter;
final boolean searchEnabled;
final DelayLimiter<AutocompleteContext> delayLimiter;
@@ -57,12 +59,18 @@ class ElasticsearchSpanConsumer implements SpanConsumer { // not final for testi
this.es = es;
this.autocompleteKeys = new LinkedHashSet<>(es.autocompleteKeys());
this.indexNameFormatter = es.indexNameFormatter();
+ this.indexTypeDelimiter = es.indexTypeDelimiter();
this.searchEnabled = es.searchEnabled();
this.delayLimiter = DelayLimiter.newBuilder()
.ttl(es.autocompleteTtl())
.cardinality(es.autocompleteCardinality()).build();
}
+ String formatTypeAndTimestampForInsert(String type, long timestampMillis) {
+ return indexNameFormatter.formatTypeAndTimestampForInsert(type, indexTypeDelimiter,
+ timestampMillis);
+ }
+
@Override public Call<Void> accept(List<Span> spans) {
if (spans.isEmpty()) return Call.create(null);
BulkSpanIndexer indexer = new BulkSpanIndexer(this);
@@ -104,8 +112,7 @@ class ElasticsearchSpanConsumer implements SpanConsumer { // not final for testi
}
void add(long indexTimestamp, Span span, long timestampMillis) {
- String index = consumer.indexNameFormatter
- .formatTypeAndTimestamp(SPAN, indexTimestamp);
+ String index = consumer.formatTypeAndTimestampForInsert(SPAN, indexTimestamp);
byte[] document = consumer.searchEnabled
? prefixWithTimestampMillisAndQuery(span, timestampMillis)
: SpanBytesEncoder.JSON_V2.encode(span);
@@ -113,7 +120,7 @@ class ElasticsearchSpanConsumer implements SpanConsumer { // not final for testi
}
void addAutocompleteValues(long indexTimestamp, Span span) {
- String idx = consumer.indexNameFormatter.formatTypeAndTimestamp(AUTOCOMPLETE, indexTimestamp);
+ String idx = consumer.formatTypeAndTimestampForInsert(AUTOCOMPLETE, indexTimestamp);
for (Map.Entry<String, String> tag : span.tags().entrySet()) {
int length = tag.getKey().length() + tag.getValue().length() + 1;
if (length > INDEX_CHARS_LIMIT) continue;
diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java
index 93f5cb6..a19541f 100644
--- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java
+++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/ElasticsearchStorage.java
@@ -29,7 +29,6 @@ import okhttp3.HttpUrl;
import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Request;
-import okhttp3.RequestBody;
import okio.Buffer;
import okio.BufferedSource;
import zipkin2.CheckResult;
@@ -46,6 +45,7 @@ import zipkin2.storage.StorageComponent;
import static zipkin2.elasticsearch.ElasticsearchAutocompleteTags.AUTOCOMPLETE;
import static zipkin2.elasticsearch.ElasticsearchSpanStore.DEPENDENCY;
import static zipkin2.elasticsearch.ElasticsearchSpanStore.SPAN;
+import static zipkin2.elasticsearch.EnsureIndexTemplate.ensureIndexTemplate;
import static zipkin2.elasticsearch.internal.JsonReaders.enterPath;
@AutoValue
@@ -145,7 +145,11 @@ public abstract class ElasticsearchStorage extends zipkin2.storage.StorageCompon
*/
public abstract Builder namesLookback(int namesLookback);
- /** Visible for testing */
+ /**
+ * Internal and visible only for testing.
+ *
+ * <p>See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-refresh.html
+ */
public abstract Builder flushOnWrites(boolean flushOnWrites);
/** The index prefix to use when generating daily index names. Defaults to zipkin. */
@@ -158,9 +162,9 @@ public abstract class ElasticsearchStorage extends zipkin2.storage.StorageCompon
* The date separator to use when generating daily index names. Defaults to '-'.
*
* <p>By default, spans with a timestamp falling on 2016/03/19 end up in the index
- * 'zipkin:span-2016-03-19'. When the date separator is '.', the index would be
- * 'zipkin:span-2016.03.19'. If the date separator is 0, there is no delimiter. Ex the index
- * would be 'zipkin:span-20160319'
+ * 'zipkin-span-2016-03-19'. When the date separator is '.', the index would be
+ * 'zipkin-span-2016.03.19'. If the date separator is 0, there is no delimiter. Ex the index
+ * would be 'zipkin-span-20160319'
*/
public final Builder dateSeparator(char dateSeparator) {
indexNameFormatterBuilder().dateSeparator(dateSeparator);
@@ -275,7 +279,11 @@ public abstract class ElasticsearchStorage extends zipkin2.storage.StorageCompon
return ensureIndexTemplates().version();
}
- /** This is a blocking call, only used in tests. */
+ char indexTypeDelimiter() {
+ return ensureIndexTemplates().indexTypeDelimiter();
+ }
+
+ /** This is an internal blocking call, only used in tests. */
public void clear() throws IOException {
Set<String> toClear = new LinkedHashSet<>();
toClear.add(indexNameFormatter().formatType(SPAN));
@@ -284,29 +292,10 @@ public abstract class ElasticsearchStorage extends zipkin2.storage.StorageCompon
}
void clear(String index) throws IOException {
- Request deleteRequest =
- new Request.Builder()
- .url(http().baseUrl.newBuilder().addPathSegment(index).build())
- .delete()
- .tag("delete-index")
- .build();
-
- http().newCall(deleteRequest, BodyConverters.NULL).execute();
-
- flush(http(), index);
- }
-
- /** This is a blocking call, only used in tests. */
- public static void flush(HttpCall.Factory factory, String index) throws IOException {
- Request flushRequest =
- new Request.Builder()
- .url(
- factory.baseUrl.newBuilder().addPathSegment(index).addPathSegment("_flush").build())
- .post(RequestBody.create(APPLICATION_JSON, ""))
- .tag("flush-index")
- .build();
-
- factory.newCall(flushRequest, BodyConverters.NULL).execute();
+ HttpUrl.Builder url = http().baseUrl.newBuilder().addPathSegment(index);
+ //if (version() >= 6.0 ) url.addQueryParameter("refresh", "wait_for");
+ Request delete = new Request.Builder().url(url.build()).delete().tag("delete-index").build();
+ http().newCall(delete, BodyConverters.NULL).execute();
}
/** This is blocking so that we can determine if the cluster is healthy or not */
@@ -334,7 +323,7 @@ public abstract class ElasticsearchStorage extends zipkin2.storage.StorageCompon
@Override
public CheckResult convert(BufferedSource b) throws IOException {
b.request(Long.MAX_VALUE); // Buffer the entire body.
- Buffer body = b.buffer();
+ Buffer body = b.getBuffer();
JsonReader status = enterPath(JsonReader.of(body.clone()), "status");
if (status == null) {
throw new IllegalStateException("Health status couldn't be read " + body.readUtf8());
@@ -353,23 +342,28 @@ public abstract class ElasticsearchStorage extends zipkin2.storage.StorageCompon
@Memoized // since we don't want overlapping calls to apply the index templates
IndexTemplates ensureIndexTemplates() {
- String index = indexNameFormatter().index();
try {
IndexTemplates templates = new VersionSpecificTemplates(this).get(http());
- EnsureIndexTemplate.apply(http(), index + ":" + SPAN + "_template", templates.span());
- EnsureIndexTemplate.apply(
- http(), index + ":" + DEPENDENCY + "_template", templates.dependency());
- EnsureIndexTemplate.apply(
- http(), index + ":" + AUTOCOMPLETE + "_template", templates.autocomplete());
+ HttpCall.Factory http = http();
+ ensureIndexTemplate(http, buildUrl(http, templates, SPAN), templates.span());
+ ensureIndexTemplate(http, buildUrl(http, templates, DEPENDENCY), templates.dependency());
+ ensureIndexTemplate(http, buildUrl(http, templates, AUTOCOMPLETE), templates.autocomplete());
return templates;
} catch (IOException e) {
throw Platform.get().uncheckedIOException(e);
}
}
- @Memoized
- public // hosts resolution might imply a network call, and we might make a new okhttp instance
- HttpCall.Factory http() {
+ HttpUrl buildUrl(HttpCall.Factory http, IndexTemplates templates, String type) {
+ HttpUrl.Builder builder = http.baseUrl.newBuilder("_template");
+ // ES 7.x defaults include_type_name to false https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#_literal_include_type_name_literal_now_defaults_to_literal_false_literal
+ if (templates.version() >= 7) builder.addQueryParameter("include_type_name", "true");
+ String indexPrefix = indexNameFormatter().index() + templates.indexTypeDelimiter();
+ return builder.addPathSegment(indexPrefix + type + "_template").build();
+ }
+
+ @Memoized // hosts resolution might imply a network call, and we might make a new okhttp instance
+ public HttpCall.Factory http() {
List<String> hosts = hostsSupplier().get();
if (hosts.isEmpty()) throw new IllegalArgumentException("no hosts configured");
OkHttpClient ok =
diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/EnsureIndexTemplate.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/EnsureIndexTemplate.java
index 68f12fe..302688b 100644
--- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/EnsureIndexTemplate.java
+++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/EnsureIndexTemplate.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import okhttp3.HttpUrl;
import okhttp3.Request;
import okhttp3.RequestBody;
-import zipkin2.elasticsearch.internal.client.HttpCall;
+import zipkin2.elasticsearch.internal.client.HttpCall.Factory;
/** Ensures the index template exists and saves off the version */
final class EnsureIndexTemplate {
@@ -29,19 +29,17 @@ final class EnsureIndexTemplate {
* This is a blocking call, used inside a lazy. That's because no writes should occur until the
* template is available.
*/
- static void apply(HttpCall.Factory callFactory, String name, String indexTemplate)
- throws IOException {
- HttpUrl templateUrl = callFactory.baseUrl.newBuilder("_template").addPathSegment(name).build();
+ static void ensureIndexTemplate(Factory callFactory, HttpUrl templateUrl, String indexTemplate)
+ throws IOException {
Request getTemplate = new Request.Builder().url(templateUrl).tag("get-template").build();
try {
callFactory.newCall(getTemplate, BodyConverters.NULL).execute();
} catch (IllegalStateException e) { // TODO: handle 404 slightly more nicely
- Request updateTemplate =
- new Request.Builder()
- .url(templateUrl)
- .put(RequestBody.create(ElasticsearchStorage.APPLICATION_JSON, indexTemplate))
- .tag("update-template")
- .build();
+ Request updateTemplate = new Request.Builder()
+ .url(templateUrl)
+ .put(RequestBody.create(ElasticsearchStorage.APPLICATION_JSON, indexTemplate))
+ .tag("update-template")
+ .build();
callFactory.newCall(updateTemplate, BodyConverters.NULL).execute();
}
}
diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java
index 5b52235..41e6f8c 100644
--- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java
+++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/IndexTemplates.java
@@ -32,6 +32,15 @@ abstract class IndexTemplates {
abstract String autocomplete();
+ /**
+ * This returns a delimiter based on what's supported by the Elasticsearch version.
+ *
+ * <p>See https://github.com/openzipkin/zipkin/issues/2219
+ */
+ char indexTypeDelimiter() {
+ return version() < 7 ? ':' : '-';
+ }
+
@AutoValue.Builder
interface Builder {
Builder version(float version);
diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java
index c49f170..b48a9a1 100644
--- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java
+++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/VersionSpecificTemplates.java
@@ -23,9 +23,9 @@ import okhttp3.Request;
import okio.BufferedSource;
import zipkin2.elasticsearch.internal.client.HttpCall;
+import static zipkin2.elasticsearch.ElasticsearchAutocompleteTags.AUTOCOMPLETE;
import static zipkin2.elasticsearch.ElasticsearchSpanStore.DEPENDENCY;
import static zipkin2.elasticsearch.ElasticsearchSpanStore.SPAN;
-import static zipkin2.elasticsearch.ElasticsearchAutocompleteTags.AUTOCOMPLETE;
import static zipkin2.elasticsearch.internal.JsonReaders.enterPath;
/** Returns a version-specific span and dependency index template */
@@ -91,9 +91,11 @@ final class VersionSpecificTemplates {
+ " },\n";
if (searchEnabled) {
return result
- + (" \"mappings\": {\n"
- + " \"_default_\": {\n"
- + " DISABLE_ALL" // don't concat all fields into big string
+ + (" \"mappings\": {\nDISABLE_ALL"
+ + " \""
+ + SPAN
+ + "\": {\n"
+ + " \"_source\": {\"excludes\": [\"_q\"] },\n"
+ " \"dynamic_templates\": [\n"
+ " {\n"
+ " \"strings\": {\n"
@@ -105,12 +107,7 @@ final class VersionSpecificTemplates {
+ " \"match\": \"*\"\n"
+ " }\n"
+ " }\n"
- + " ]\n"
- + " },\n"
- + " \""
- + SPAN
- + "\": {\n"
- + " \"_source\": {\"excludes\": [\"_q\"] },\n"
+ + " ],\n"
+ " \"properties\": {\n"
+ " \"traceId\": ${__TRACE_ID_MAPPING__},\n"
+ " \"name\": { KEYWORD },\n"
@@ -138,8 +135,7 @@ final class VersionSpecificTemplates {
+ "}");
}
return result
- + (" \"mappings\": {\n"
- + " \"_default_\": { DISABLE_ALL },\n"
+ + (" \"mappings\": {\nDISABLE_ALL"
+ " \""
+ SPAN
+ "\": {\n"
@@ -181,12 +177,12 @@ final class VersionSpecificTemplates {
+ " \"index.number_of_shards\": ${__NUMBER_OF_SHARDS__},\n"
+ " \"index.number_of_replicas\": ${__NUMBER_OF_REPLICAS__},\n"
+ " \"index.requests.cache.enable\": true,\n"
- + " \"index.mapper.dynamic\": true\n"
+ + " \"index.mapper.dynamic\": false\n"
+ " },\n"
+ " \"mappings\": {\""
+ AUTOCOMPLETE
+ "\": { \"enabled\": true,\n"
- + " \t\"properties\": {\n"
+ + " \"properties\": {\n"
+ " \"tagKey\": { KEYWORD },\n"
+ " \"tagValue\": { KEYWORD }\n"
+ " }}}\n"
@@ -228,47 +224,62 @@ final class VersionSpecificTemplates {
}
private String versionSpecificSpanIndexTemplate(float version) {
+ String result;
if (version >= 2 && version < 3) {
- return spanIndexTemplate
+ result = spanIndexTemplate
.replace("TEMPLATE", "template")
.replace("STRING", "string")
- .replace("DISABLE_ALL", "\"_all\": {\"enabled\": false}" + (searchEnabled ? ",\n" : ""))
+ .replace("DISABLE_ALL", "\"_default_\": { \"_all\": {\"enabled\": false} },\n")
.replace(
"KEYWORD",
"\"type\": \"string\", \"norms\": {\"enabled\": false }, \"index\": \"not_analyzed\"");
} else if (version >= 5) {
- return spanIndexTemplate
+ result = spanIndexTemplate
.replace("TEMPLATE", version >= 6 ? "index_patterns" : "template")
.replace("STRING", "text")
- .replace("DISABLE_ALL", "") // _all isn't supported in 6.x anyway
+ // 6.x _all disabled https://www.elastic.co/guide/en/elasticsearch/reference/6.7/breaking-changes-6.0.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default
+ // 7.x _default disallowed https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#_the_literal__default__literal_mapping_is_no_longer_allowed
+ .replace("DISABLE_ALL", "")
.replace("KEYWORD", "\"type\": \"keyword\", \"norms\": false")
.replace(
"\"analyzer\": \"traceId_analyzer\" }",
"\"fielddata\": \"true\", \"analyzer\": \"traceId_analyzer\" }");
} else {
- throw new IllegalStateException(
- "Elasticsearch 2.x, 5.x and 6.x are supported, was: " + version);
+ throw new IllegalStateException("Elasticsearch 2-7.x are supported, was: " + version);
}
+ return maybeReviseFor7x(SPAN, version, result);
}
private String versionSpecificDependencyLinkIndexTemplate(float version) {
- return dependencyIndexTemplate.replace(
- "TEMPLATE", version >= 6 ? "index_patterns" : "template");
+ String result = dependencyIndexTemplate.replace(
+ "TEMPLATE", version >= 6 ? "index_patterns" : "template");
+ return maybeReviseFor7x(DEPENDENCY, version, result);
}
+
private String versionSpecificAutocompleteIndexTemplate(float version) {
+ String result;
if (version >= 2 && version < 3) {
- return autocompleteIndexTemplate
+ result = autocompleteIndexTemplate
.replace("TEMPLATE", "template")
.replace("KEYWORD", "\"type\": \"string\", \"norms\": {\"enabled\": false }, \"index\": "
+ "\"not_analyzed\"");
} else if (version >= 5) {
- return autocompleteIndexTemplate
+ result = autocompleteIndexTemplate
.replace("TEMPLATE", version >= 6 ? "index_patterns" : "template")
.replace("KEYWORD", "\"type\": \"keyword\",\"norms\": false\n");
- }else {
- throw new IllegalStateException(
- "Elasticsearch 2.x, 5.x and 6.x are supported, was: " + version);
+ } else {
+ throw new IllegalStateException("Elasticsearch 2-7.x are supported, was: " + version);
}
+ return maybeReviseFor7x(AUTOCOMPLETE, version, result);
+ }
+
+ private String maybeReviseFor7x(String type, float version, String result) {
+ if (version < 7) return result;
+ // Colons are no longer allowed in index names. Make sure the pattern in our index template
+ // doesn't use them either.
+ result = result.replaceAll(":" + type, "-" + type);
+ result = result.replaceAll(",\n +\"index\\.mapper\\.dynamic\": false", "");
+ return result;
}
}
diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/HttpBulkIndexer.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/HttpBulkIndexer.java
index 08d21f5..c52cda1 100644
--- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/HttpBulkIndexer.java
+++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/HttpBulkIndexer.java
@@ -17,10 +17,6 @@
package zipkin2.elasticsearch.internal;
import java.io.IOException;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.Set;
import java.util.concurrent.RejectedExecutionException;
import okhttp3.HttpUrl;
import okhttp3.MediaType;
@@ -42,34 +38,16 @@ public final class HttpBulkIndexer {
final String tag;
final HttpCall.Factory http;
final String pipeline;
- final boolean flushOnWrites;
+ final boolean waitForRefresh;
// Mutated for each call to add
final Buffer body = new Buffer();
- final Set<String> indices;
- final HttpCall.BodyConverter<Void> maybeFlush;
public HttpBulkIndexer(String tag, ElasticsearchStorage es) {
this.tag = tag;
http = es.http();
pipeline = es.pipeline();
- flushOnWrites = es.flushOnWrites();
- if (flushOnWrites) {
- indices = new LinkedHashSet<>();
- maybeFlush =
- new HttpCall.BodyConverter<Void>() {
- @Override
- public Void convert(BufferedSource b) throws IOException {
- CheckForErrors.INSTANCE.convert(b);
- if (indices.isEmpty()) return null;
- ElasticsearchStorage.flush(http, join(indices));
- return null;
- }
- };
- } else {
- indices = null;
- maybeFlush = CheckForErrors.INSTANCE;
- }
+ waitForRefresh = es.flushOnWrites();
}
enum CheckForErrors implements HttpCall.BodyConverter<Void> {
@@ -95,7 +73,6 @@ public final class HttpBulkIndexer {
}
void writeIndexMetadata(String index, String typeName, @Nullable String id) {
- if (flushOnWrites) indices.add(index);
body.writeUtf8("{\"index\":{\"_index\":\"").writeUtf8(index).writeByte('"');
// the _type parameter is needed for Elasticsearch <6.x
body.writeUtf8(",\"_type\":\"").writeUtf8(typeName).writeByte('"');
@@ -112,27 +89,16 @@ public final class HttpBulkIndexer {
/** Creates a bulk request when there is more than one object to store */
public HttpCall<Void> newCall() {
- HttpUrl url =
- pipeline != null
- ? http.baseUrl.newBuilder("_bulk").addQueryParameter("pipeline", pipeline).build()
- : http.baseUrl.resolve("_bulk");
-
- Request request =
- new Request.Builder()
- .url(url)
- .tag(tag)
- .post(RequestBody.create(APPLICATION_JSON, body.readByteString()))
- .build();
+ HttpUrl.Builder urlBuilder = http.baseUrl.newBuilder("_bulk");
+ if (pipeline != null) urlBuilder.addQueryParameter("pipeline", pipeline);
+ if (waitForRefresh) urlBuilder.addQueryParameter("refresh", "wait_for");
- return http.newCall(request, maybeFlush);
- }
+ Request request = new Request.Builder()
+ .url(urlBuilder.build())
+ .tag(tag)
+ .post(RequestBody.create(APPLICATION_JSON, body.readByteString()))
+ .build();
- static String join(Collection<String> parts) {
- Iterator<String> iterator = parts.iterator();
- StringBuilder result = new StringBuilder(iterator.next());
- while (iterator.hasNext()) {
- result.append(',').append(iterator.next());
- }
- return result.toString();
+ return http.newCall(request, CheckForErrors.INSTANCE);
}
}
diff --git a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/IndexNameFormatter.java b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/IndexNameFormatter.java
index 5f21779..4b70737 100644
--- a/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/IndexNameFormatter.java
+++ b/zipkin-storage/elasticsearch/src/main/java/zipkin2/elasticsearch/internal/IndexNameFormatter.java
@@ -31,6 +31,44 @@ import zipkin2.internal.Nullable;
import static java.lang.String.format;
+/**
+ * <h3>Index-Prefix/type delimiter</h3>
+ * When Elasticsearch dropped support for multiple type indexes, we introduced a delimited naming
+ * convention to distinguish between span, dependency and autocomplete documents. Originally, this
+ * was a colon prefix pattern. In version 7, Elasticsearch dropped support for colons in indexes. To
+ * keep existing writes consistent, we still use colon in versions prior to ES 7, eventhough
+ * starting at version 7, we change to hyphens. {@code zipkin2.elasticsearch.IndexTemplates} is
+ * responsible for this decision.
+ *
+ * <p><h3>Creating indexes</h3>
+ * Using the default index prefix of "zipkin", when indexes are created, they look like the
+ * following, based on the version.
+ *
+ * <ul>
+ * <li>ES up to v6: zipkin:span-2019-05-03 zipkin:dependency-2019-05-03 zipkin:autocomplete-2019-05-03</li>
+ * <li>ES v7: zipkin-span-2019-05-03 zipkin-dependency-2019-05-03 zipkin-autocomplete-2019-05-03</li>
+ * </ul>
+ *
+ * <p>We can allow an index prefix of up to 231 UTF-8 encoded bytes, subject to the index naming
+ * constraints. This is the normal 255 limit minus the longest suffix (ex. -autocomplete-2019-05-03).
+ *
+ * <p><h3>Reading indexes</h3>
+ * While ES 7 cannot write new indexes with a colons, it can read them. Upon upgrade, some sites
+ * will have a mixed read state where some indexes delimit types with a colon and others a hyphen.
+ * Accordingly, we use * in read patterns in place of a type delimiter. We use * because there is no
+ * support for single character wildcards in ES.
+ *
+ * <p><h3>Elasticsearch 7 naming constraints</h3>
+ * According to a <a href="https://github.com/elastic/elasticsearch/blob/83e9d0b9c63589f1dc5bda8abb6b10b27502ef71/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java#L162">recent
+ * reference</a>, the following index naming constraints apply to index names as of ES 7:
+ *
+ * <ul>
+ * <li>No more than 255 UTF-8 encoded bytes</li>
+ * <li>Cannot be . or ..</li>
+ * <li>Cannot contain : or #</li>
+ * <li>Cannot start with _ - or +</li>
+ * </ul>
+ */
@AutoValue
public abstract class IndexNameFormatter {
public static Builder newBuilder() {
@@ -157,12 +195,21 @@ public abstract class IndexNameFormatter {
return result;
}
+ /** On insert, require a version-specific index-type delimiter as ES 7+ dropped colons */
+ public String formatTypeAndTimestampForInsert(String type, char indexTypeDelimiter,
+ long timestampMillis) {
+ return index() + indexTypeDelimiter + type + '-' + dateFormat().get()
+ .format(new Date(timestampMillis));
+ }
+
public String formatTypeAndTimestamp(@Nullable String type, long timestampMillis) {
return prefix(type) + "-" + dateFormat().get().format(new Date(timestampMillis));
}
private String prefix(@Nullable String type) {
- return type != null ? index() + ":" + type : index();
+ // We use single-character wildcard here in order to read both : and - as starting in ES 7, :
+ // is no longer permitted.
+ return type != null ? index() + "*" + type : index();
}
// for testing
diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchSpanConsumerTest.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchSpanConsumerTest.java
index 7cdc812..f74fae8 100644
--- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchSpanConsumerTest.java
+++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchSpanConsumerTest.java
@@ -335,7 +335,6 @@ public class ElasticsearchSpanConsumerTest {
.contains(
""
+ " \"mappings\": {\n"
- + " \"_default_\": { },\n"
+ " \"span\": {\n"
+ " \"properties\": {\n"
+ " \"traceId\": { \"type\": \"keyword\", \"norms\": false },\n"
diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchStorageTest.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchStorageTest.java
index 4edbcb6..0609fe3 100644
--- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchStorageTest.java
+++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/ElasticsearchStorageTest.java
@@ -60,9 +60,9 @@ public class ElasticsearchStorageTest {
es.takeRequest(); // get tags template
assertThat(es.takeRequest().getPath())
- .startsWith("/zipkin:dependency-2016-10-01,zipkin:dependency-2016-10-02/_search");
+ .startsWith("/zipkin*dependency-2016-10-01,zipkin*dependency-2016-10-02/_search");
assertThat(es.takeRequest().getPath())
- .startsWith("/zipkin:dependency-2016-10-01,zipkin:dependency-2016-10-02/_search");
+ .startsWith("/zipkin*dependency-2016-10-01,zipkin*dependency-2016-10-02/_search");
}
String healthResponse =
diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/InternalForTests.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/InternalForTests.java
index 2b67613..c38b8eb 100644
--- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/InternalForTests.java
+++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/InternalForTests.java
@@ -27,8 +27,8 @@ import zipkin2.elasticsearch.internal.HttpBulkIndexer;
public class InternalForTests {
public static void writeDependencyLinks(ElasticsearchStorage es, List<DependencyLink> links,
long midnightUTC) {
- String index =
- es.indexNameFormatter().formatTypeAndTimestamp("dependency", midnightUTC);
+ String index = ((ElasticsearchSpanConsumer) es.spanConsumer())
+ .formatTypeAndTimestampForInsert("dependency", midnightUTC);
HttpBulkIndexer indexer = new HttpBulkIndexer("indexlinks", es);
for (DependencyLink link : links) {
byte[] document = DependencyLinkBytesEncoder.JSON_V1.encode(link);
diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/VersionSpecificTemplatesTest.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/VersionSpecificTemplatesTest.java
index 1161011..d747ffa 100644
--- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/VersionSpecificTemplatesTest.java
+++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/VersionSpecificTemplatesTest.java
@@ -16,7 +16,6 @@
*/
package zipkin2.elasticsearch;
-import java.io.IOException;
import okhttp3.mockwebserver.MockResponse;
import okhttp3.mockwebserver.MockWebServer;
import org.junit.After;
@@ -34,8 +33,6 @@ public class VersionSpecificTemplatesTest {
ElasticsearchStorage storage =
ElasticsearchStorage.newBuilder().hosts(asList(es.url("").toString())).build();
- VersionSpecificTemplates client = new VersionSpecificTemplates(storage);
-
@After
public void close() {
storage.close();
@@ -128,4 +125,50 @@ public class VersionSpecificTemplatesTest {
assertThat(VersionSpecificTemplates.getVersion(storage.http())).isEqualTo(6.0f);
}
+
+ @Test public void getVersion_6_7() throws Exception {
+ es.enqueue(new MockResponse().setBody(
+ "{\n"
+ + " \"name\" : \"PV-NhJd\",\n"
+ + " \"cluster_name\" : \"CollectorDBCluster\",\n"
+ + " \"cluster_uuid\" : \"UjZaM0fQRC6tkHINCg9y8w\",\n"
+ + " \"version\" : {\n"
+ + " \"number\" : \"6.7.0\",\n"
+ + " \"build_flavor\" : \"oss\",\n"
+ + " \"build_type\" : \"tar\",\n"
+ + " \"build_hash\" : \"8453f77\",\n"
+ + " \"build_date\" : \"2019-03-21T15:32:29.844721Z\",\n"
+ + " \"build_snapshot\" : false,\n"
+ + " \"lucene_version\" : \"7.7.0\",\n"
+ + " \"minimum_wire_compatibility_version\" : \"5.6.0\",\n"
+ + " \"minimum_index_compatibility_version\" : \"5.0.0\"\n"
+ + " },\n"
+ + " \"tagline\" : \"You Know, for Search\"\n"
+ + "}"));
+
+ assertThat(VersionSpecificTemplates.getVersion(storage.http())).isEqualTo(6.7f);
+ }
+
+ @Test public void getVersion_7() throws Exception {
+ es.enqueue(new MockResponse().setBody(
+ "{\n"
+ + " \"name\" : \"zipkin-elasticsearch\",\n"
+ + " \"cluster_name\" : \"docker-cluster\",\n"
+ + " \"cluster_uuid\" : \"wByRPgSgTryYl0TZXW4MsA\",\n"
+ + " \"version\" : {\n"
+ + " \"number\" : \"7.0.1\",\n"
+ + " \"build_flavor\" : \"default\",\n"
+ + " \"build_type\" : \"tar\",\n"
+ + " \"build_hash\" : \"e4efcb5\",\n"
+ + " \"build_date\" : \"2019-04-29T12:56:03.145736Z\",\n"
+ + " \"build_snapshot\" : false,\n"
+ + " \"lucene_version\" : \"8.0.0\",\n"
+ + " \"minimum_wire_compatibility_version\" : \"6.7.0\",\n"
+ + " \"minimum_index_compatibility_version\" : \"6.0.0-beta1\"\n"
+ + " },\n"
+ + " \"tagline\" : \"You Know, for Search\"\n"
+ + "}"));
+
+ assertThat(VersionSpecificTemplates.getVersion(storage.http())).isEqualTo(7.0f);
+ }
}
diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchStorageRule.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchStorageRule.java
index b397c94..d42bb34 100644
--- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchStorageRule.java
+++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ElasticsearchStorageRule.java
@@ -27,7 +27,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
-import org.testcontainers.containers.wait.HttpWaitStrategy;
+import org.testcontainers.containers.wait.strategy.HttpWaitStrategy;
import zipkin2.CheckResult;
import zipkin2.elasticsearch.ElasticsearchStorage;
diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV7.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV7.java
new file mode 100644
index 0000000..e2c37f3
--- /dev/null
+++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/integration/ITElasticsearchStorageV7.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package zipkin2.elasticsearch.integration;
+
+import java.io.IOException;
+import java.util.List;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+import zipkin2.Span;
+import zipkin2.elasticsearch.ElasticsearchStorage;
+import zipkin2.elasticsearch.InternalForTests;
+import zipkin2.storage.StorageComponent;
+
+import static zipkin2.elasticsearch.integration.ElasticsearchStorageRule.index;
+
+@RunWith(Enclosed.class)
+public class ITElasticsearchStorageV7 {
+
+ static ElasticsearchStorageRule classRule() {
+ return new ElasticsearchStorageRule("openzipkin/zipkin-elasticsearch7:2.12.9",
+ "test_elasticsearch3");
+ }
+
+ public static class ITSpanStore extends zipkin2.storage.ITSpanStore {
+ @ClassRule public static ElasticsearchStorageRule backend = classRule();
+ @Rule public TestName testName = new TestName();
+
+ ElasticsearchStorage storage;
+
+ @Before public void connect() {
+ storage = backend.computeStorageBuilder().index(index(testName)).build();
+ }
+
+ @Override protected StorageComponent storage() {
+ return storage;
+ }
+
+ @Override @Test @Ignore("No consumer-side span deduplication") public void deduplicates() {
+ }
+
+ @Before @Override public void clear() throws IOException {
+ storage.clear();
+ }
+ }
+
+ public static class ITSearchEnabledFalse extends zipkin2.storage.ITSearchEnabledFalse {
+ @ClassRule public static ElasticsearchStorageRule backend = classRule();
+ @Rule public TestName testName = new TestName();
+
+ ElasticsearchStorage storage;
+
+ @Before public void connect() {
+ storage = backend.computeStorageBuilder().index(index(testName))
+ .searchEnabled(false).build();
+ }
+
+ @Override protected StorageComponent storage() {
+ return storage;
+ }
+
+ @Before @Override public void clear() throws IOException {
+ storage.clear();
+ }
+ }
+
+ public static class ITAutocompleteTags extends zipkin2.storage.ITAutocompleteTags {
+ @ClassRule public static ElasticsearchStorageRule backend = classRule();
+ @Rule public TestName testName = new TestName();
+
+ @Override protected StorageComponent.Builder storageBuilder() {
+ return backend.computeStorageBuilder().index(index(testName));
+ }
+
+ @Before @Override public void clear() throws IOException {
+ ((ElasticsearchStorage) storage).clear();
+ }
+ }
+
+ public static class ITStrictTraceIdFalse extends zipkin2.storage.ITStrictTraceIdFalse {
+ @ClassRule public static ElasticsearchStorageRule backend = classRule();
+ @Rule public TestName testName = new TestName();
+
+ ElasticsearchStorage storage;
+
+ @Before public void connect() {
+ storage = backend.computeStorageBuilder().index(index(testName)).strictTraceId(false).build();
+ }
+
+ @Override protected StorageComponent storage() {
+ return storage;
+ }
+
+ @Before @Override public void clear() throws IOException {
+ storage.clear();
+ }
+ }
+
+ public static class ITDependencies extends zipkin2.storage.ITDependencies {
+ @ClassRule public static ElasticsearchStorageRule backend = classRule();
+ @Rule public TestName testName = new TestName();
+
+ ElasticsearchStorage storage;
+
+ @Before public void connect() {
+ storage = backend.computeStorageBuilder().index(index(testName)).build();
+ }
+
+ @Override protected StorageComponent storage() {
+ return storage;
+ }
+
+ /**
+ * The current implementation does not include dependency aggregation. It includes retrieval of
+ * pre-aggregated links, usually made via zipkin-dependencies
+ */
+ @Override protected void processDependencies(List<Span> spans) throws Exception {
+ aggregateLinks(spans).forEach(
+ (midnight, links) -> InternalForTests.writeDependencyLinks(storage, links, midnight));
+ }
+
+ @Before @Override public void clear() throws IOException {
+ storage.clear();
+ }
+ }
+}
diff --git a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/internal/IndexNameFormatterTest.java b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/internal/IndexNameFormatterTest.java
index fb1ccf1..c867f63 100644
--- a/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/internal/IndexNameFormatterTest.java
+++ b/zipkin-storage/elasticsearch/src/test/java/zipkin2/elasticsearch/internal/IndexNameFormatterTest.java
@@ -39,7 +39,7 @@ public class IndexNameFormatterTest {
long end = iso8601.parse("2016-11-01T23:59:59Z").getTime();
assertThat(formatter.formatTypeAndRange("span", start, end))
- .containsExactly("zipkin:span-2016-11-01");
+ .containsExactly("zipkin*span-2016-11-01");
}
@Test
@@ -48,7 +48,7 @@ public class IndexNameFormatterTest {
long end = iso8601.parse("2016-11-16T23:59:59Z").getTime();
assertThat(formatter.formatTypeAndRange("span", start, end))
- .containsExactly("zipkin:span-2016-11-15", "zipkin:span-2016-11-16");
+ .containsExactly("zipkin*span-2016-11-15", "zipkin*span-2016-11-16");
}
@Test
@@ -58,7 +58,7 @@ public class IndexNameFormatterTest {
assertThat(formatter.formatTypeAndRange("span", start, end))
.containsExactly(
- "zipkin:span-2016-11-01", "zipkin:span-2016-11-02", "zipkin:span-2016-11-03");
+ "zipkin*span-2016-11-01", "zipkin*span-2016-11-02", "zipkin*span-2016-11-03");
}
@Test
@@ -67,7 +67,7 @@ public class IndexNameFormatterTest {
long end = iso8601.parse("2016-11-01T23:59:59Z").getTime();
assertThat(formatter.formatTypeAndRange("span", start, end))
- .containsExactly("zipkin:span-2016-10-31", "zipkin:span-2016-11-01");
+ .containsExactly("zipkin*span-2016-10-31", "zipkin*span-2016-11-01");
}
@Test
@@ -76,7 +76,7 @@ public class IndexNameFormatterTest {
long end = iso8601.parse("2016-10-31T23:59:59Z").getTime();
assertThat(formatter.formatTypeAndRange("span", start, end))
- .containsExactly("zipkin:span-2016-10-*");
+ .containsExactly("zipkin*span-2016-10-*");
}
@Test
@@ -86,7 +86,7 @@ public class IndexNameFormatterTest {
assertThat(formatter.formatTypeAndRange("span", start, end))
.containsExactly(
- "zipkin:span-2016-10-31", "zipkin:span-2016-11-*", "zipkin:span-2016-12-01");
+ "zipkin*span-2016-10-31", "zipkin*span-2016-11-*", "zipkin*span-2016-12-01");
}
@Test
@@ -96,10 +96,10 @@ public class IndexNameFormatterTest {
assertThat(formatter.formatTypeAndRange("span", start, end))
.containsExactly(
- "zipkin:span-2016-02-28",
- "zipkin:span-2016-02-29",
- "zipkin:span-2016-03-*",
- "zipkin:span-2016-04-01");
+ "zipkin*span-2016-02-28",
+ "zipkin*span-2016-02-29",
+ "zipkin*span-2016-03-*",
+ "zipkin*span-2016-04-01");
}
@Test
@@ -108,7 +108,7 @@ public class IndexNameFormatterTest {
long end = iso8601.parse("2016-12-31T23:59:59Z").getTime();
assertThat(formatter.formatTypeAndRange("span", start, end))
- .containsExactly("zipkin:span-2016-*");
+ .containsExactly("zipkin*span-2016-*");
}
@Test
@@ -118,11 +118,11 @@ public class IndexNameFormatterTest {
assertThat(formatter.formatTypeAndRange("span", start, end))
.containsExactly(
- "zipkin:span-2016-10-31",
- "zipkin:span-2016-11-*",
- "zipkin:span-2016-12-*",
- "zipkin:span-2017-*",
- "zipkin:span-2018-01-01");
+ "zipkin*span-2016-10-31",
+ "zipkin*span-2016-11-*",
+ "zipkin*span-2016-12-*",
+ "zipkin*span-2017-*",
+ "zipkin*span-2018-01-01");
}
@Test
@@ -132,7 +132,7 @@ public class IndexNameFormatterTest {
long end = iso8601.parse("2016-11-01T23:59:59Z").getTime();
assertThat(formatter.formatTypeAndRange("span", start, end))
- .containsExactly("zipkin:span-2016.11.01");
+ .containsExactly("zipkin*span-2016.11.01");
}
@Test
@@ -142,7 +142,7 @@ public class IndexNameFormatterTest {
long end = iso8601.parse("2016-11-16T23:59:59Z").getTime();
assertThat(formatter.formatTypeAndRange("span", start, end))
- .containsExactly("zipkin:span-2016.11.15", "zipkin:span-2016.11.16");
+ .containsExactly("zipkin*span-2016.11.15", "zipkin*span-2016.11.16");
}
@Test
@@ -153,7 +153,7 @@ public class IndexNameFormatterTest {
assertThat(formatter.formatTypeAndRange("span", start, end))
.containsExactly(
- "zipkin:span-2016.11.01", "zipkin:span-2016.11.02", "zipkin:span-2016.11.03");
+ "zipkin*span-2016.11.01", "zipkin*span-2016.11.02", "zipkin*span-2016.11.03");
}
@Test
@@ -163,7 +163,7 @@ public class IndexNameFormatterTest {
long end = iso8601.parse("2016-11-01T23:59:59Z").getTime();
assertThat(formatter.formatTypeAndRange("span", start, end))
- .containsExactly("zipkin:span-2016.10.31", "zipkin:span-2016.11.01");
+ .containsExactly("zipkin*span-2016.10.31", "zipkin*span-2016.11.01");
}
@Test
@@ -173,7 +173,7 @@ public class IndexNameFormatterTest {
long end = iso8601.parse("2016-10-31T23:59:59Z").getTime();
assertThat(formatter.formatTypeAndRange("span", start, end))
- .containsExactly("zipkin:span-2016.10.*");
+ .containsExactly("zipkin*span-2016.10.*");
}
@Test
@@ -184,7 +184,7 @@ public class IndexNameFormatterTest {
assertThat(formatter.formatTypeAndRange("span", start, end))
.containsExactly(
- "zipkin:span-2016.10.31", "zipkin:span-2016.11.*", "zipkin:span-2016.12.01");
+ "zipkin*span-2016.10.31", "zipkin*span-2016.11.*", "zipkin*span-2016.12.01");
}
@Test
@@ -195,10 +195,10 @@ public class IndexNameFormatterTest {
assertThat(formatter.formatTypeAndRange("span", start, end))
.containsExactly(
- "zipkin:span-2016.02.28",
- "zipkin:span-2016.02.29",
- "zipkin:span-2016.03.*",
- "zipkin:span-2016.04.01");
+ "zipkin*span-2016.02.28",
+ "zipkin*span-2016.02.29",
+ "zipkin*span-2016.03.*",
+ "zipkin*span-2016.04.01");
}
@Test
@@ -208,7 +208,7 @@ public class IndexNameFormatterTest {
long end = iso8601.parse("2016-12-31T23:59:59Z").getTime();
assertThat(formatter.formatTypeAndRange("span", start, end))
- .containsExactly("zipkin:span-2016.*");
+ .containsExactly("zipkin*span-2016.*");
}
@Test
@@ -219,11 +219,11 @@ public class IndexNameFormatterTest {
assertThat(formatter.formatTypeAndRange("span", start, end))
.containsExactly(
- "zipkin:span-2016.10.31",
- "zipkin:span-2016.11.*",
- "zipkin:span-2016.12.*",
- "zipkin:span-2017.*",
- "zipkin:span-2018.01.01");
+ "zipkin*span-2016.10.31",
+ "zipkin*span-2016.11.*",
+ "zipkin*span-2016.12.*",
+ "zipkin*span-2017.*",
+ "zipkin*span-2018.01.01");
}
@Test
@@ -234,10 +234,10 @@ public class IndexNameFormatterTest {
assertThat(formatter.formatTypeAndRange("span", start, end))
.containsExactly(
- "zipkin:span-2016.10.0*",
- "zipkin:span-2016.10.1*",
- "zipkin:span-2016.10.2*",
- "zipkin:span-2016.10.30");
+ "zipkin*span-2016.10.0*",
+ "zipkin*span-2016.10.1*",
+ "zipkin*span-2016.10.2*",
+ "zipkin*span-2016.10.30");
}
@Test
@@ -248,10 +248,10 @@ public class IndexNameFormatterTest {
assertThat(formatter.formatTypeAndRange("span", start, end))
.containsExactly(
- "zipkin:span-2016.10.09",
- "zipkin:span-2016.10.1*",
- "zipkin:span-2016.10.2*",
- "zipkin:span-2016.10.30");
+ "zipkin*span-2016.10.09",
+ "zipkin*span-2016.10.1*",
+ "zipkin*span-2016.10.2*",
+ "zipkin*span-2016.10.30");
}
@Test
@@ -262,9 +262,9 @@ public class IndexNameFormatterTest {
assertThat(formatter.formatTypeAndRange("span", start, end))
.containsExactly(
- "zipkin:span-2016.10.19",
- "zipkin:span-2016.10.2*",
- "zipkin:span-2016.10.30");
+ "zipkin*span-2016.10.19",
+ "zipkin*span-2016.10.2*",
+ "zipkin*span-2016.10.30");
}
@Test
@@ -274,6 +274,6 @@ public class IndexNameFormatterTest {
long end = iso8601.parse("2016-06-30T01:01:01Z").getTime();
assertThat(formatter.formatTypeAndRange("span", start, end))
- .containsExactly("zipkin:span-2016.06.*");
+ .containsExactly("zipkin*span-2016.06.*");
}
}