You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2022/11/25 07:32:46 UTC
[skywalking] branch master updated: Fix Elasticsearch storage: In `No-Sharding Mode`, add specific analyzer to the template before index creation to avoid update index error. (#10026)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new f9e23eb329 Fix Elasticsearch storage: In `No-Sharding Mode`, add specific analyzer to the template before index creation to avoid update index error. (#10026)
f9e23eb329 is described below
commit f9e23eb32911b85a90e2d7699ee4591dd11ad1a4
Author: Wan Kai <wa...@foxmail.com>
AuthorDate: Fri Nov 25 15:32:40 2022 +0800
Fix Elasticsearch storage: In `No-Sharding Mode`, add specific analyzer to the template before index creation to avoid update index error. (#10026)
---
docs/en/changes/changes.md | 1 +
.../elasticsearch/base/StorageEsInstaller.java | 34 ++++++++++++++++++++--
2 files changed, 32 insertions(+), 3 deletions(-)
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index bbbadf7ba9..0154b9b051 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -130,6 +130,7 @@
since BanyanDB stream requires a timestamp in milliseconds.
For SQL-Database: add new column `timestamp` for tables `profile_task_log/top_n_database_statement`,
requires altering this column or removing these tables before OAP starts, if bump up from previous releases.
+* Fix Elasticsearch storage: In `No-Sharding Mode`, add specific analyzer to the template before index creation to avoid update index error.
#### UI
diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
index f514688c88..f5345a6c27 100644
--- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
+++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java
@@ -32,6 +32,7 @@ import org.apache.skywalking.library.elasticsearch.response.IndexTemplate;
import org.apache.skywalking.library.elasticsearch.response.Mappings;
import org.apache.skywalking.oap.server.core.RunningMode;
import org.apache.skywalking.oap.server.core.storage.StorageException;
+import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch;
import org.apache.skywalking.oap.server.core.storage.model.Model;
import org.apache.skywalking.oap.server.core.storage.model.ModelColumn;
import org.apache.skywalking.oap.server.core.storage.model.ModelInstaller;
@@ -248,8 +249,7 @@ public class StorageEsInstaller extends ModelInstaller {
indexRefreshInterval = 5;
}
indexSettings.put("refresh_interval", indexRefreshInterval + "s");
- List<ModelColumn> columns = IndexController.LogicIndicesRegister.getPhysicalTableColumns(model);
- indexSettings.put("analysis", getAnalyzerSetting(columns));
+ indexSettings.put("analysis", getAnalyzerSetting(model));
if (!StringUtil.isEmpty(config.getAdvanced())) {
Map<String, Object> advancedSettings = gson.fromJson(config.getAdvanced(), Map.class);
setting.putAll(advancedSettings);
@@ -264,8 +264,26 @@ public class StorageEsInstaller extends ModelInstaller {
return setting;
}
- private Map getAnalyzerSetting(List<ModelColumn> analyzerTypes) throws StorageException {
+ //In the `No-Sharding Mode`:
+ //https://skywalking.apache.org/docs/main/next/en/faq/new-elasticsearch-storage-option-explanation-in-9.2.0/
+ //Some of models require a analyzer to run match query, some others are not.
+ //They are merged into the one physical index(metrics-all or record-all)
+ //When adding a new model(with an analyzer) into an existed index by update will be failed, if the index is without analyzer settings.
+ //To avoid this, add the analyzer settings to the template before index creation.
+ private Map getAnalyzerSetting(Model model) throws StorageException {
+ if (config.isLogicSharding() || !model.isTimeSeries()) {
+ return getAnalyzerSettingByColumn(model);
+ } else if (IndexController.INSTANCE.isRecordModel(model) && model.isSuperDataset()) {
+ //SuperDataset doesn't merge index, the analyzer follow the column config.
+ return getAnalyzerSettingByColumn(model);
+ } else {
+ return getAnalyzerSetting4MergedIndex(model);
+ }
+ }
+
+ private Map getAnalyzerSettingByColumn(Model model) throws StorageException {
AnalyzerSetting analyzerSetting = new AnalyzerSetting();
+ List<ModelColumn> analyzerTypes = IndexController.LogicIndicesRegister.getPhysicalTableColumns(model);
for (final ModelColumn column : analyzerTypes) {
if (!column.getElasticSearchExtension().needMatchQuery()) {
continue;
@@ -279,6 +297,16 @@ public class StorageEsInstaller extends ModelInstaller {
return gson.fromJson(gson.toJson(analyzerSetting), Map.class);
}
+ //Indexes `metrics-all and records-all` are required `OAP_ANALYZER`
+ private Map getAnalyzerSetting4MergedIndex(Model model) throws StorageException {
+ AnalyzerSetting setting = AnalyzerSetting.Generator.getGenerator(
+ ElasticSearch.MatchQuery.AnalyzerType.OAP_ANALYZER)
+ .getGenerateFunc()
+ .generate(config);
+
+ return gson.fromJson(gson.toJson(setting), Map.class);
+ }
+
protected Mappings createMapping(Model model) {
Map<String, Object> properties = new HashMap<>();
Mappings.Source source = new Mappings.Source();