You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ni...@apache.org on 2021/04/20 11:31:34 UTC
[ignite] branch ignite-ducktape updated: IGNITE-14509 XML-based
configuration for thin client (#8998)
This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch ignite-ducktape
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-ducktape by this push:
new b52b0ac IGNITE-14509 XML-based configuration for thin client (#8998)
b52b0ac is described below
commit b52b0acf96bc80316e5d51b3414334beac9f4788
Author: eadha <58...@users.noreply.github.com>
AuthorDate: Tue Apr 20 14:31:11 2021 +0300
IGNITE-14509 XML-based configuration for thin client (#8998)
Co-authored-by: emvdovets <em...@sberbank.ru>
Co-authored-by: Nikolay <ni...@apache.org>
---
.../utils/IgniteAwareApplicationService.java | 35 ++++++++++++++--------
.../tests/ignitetest/services/ignite_app.py | 4 +--
.../tests/ignitetest/services/utils/__init__.py | 17 +++++++++++
.../ignitetest/services/utils/config_template.py | 9 ++++++
.../ignitetest/services/utils/ignite_aware.py | 4 +--
.../utils/ignite_configuration/__init__.py | 30 +++++++++++++++++++
.../tests/ignitetest/services/utils/ignite_spec.py | 30 +++++++++++++------
.../tests/ignitetest/services/utils/path.py | 15 ++++++++++
.../utils/templates/thin_client_config.xml.j2 | 29 ++++++++++++++++++
.../tests/ignitetest/tests/thin_client_test.py | 13 ++++----
10 files changed, 154 insertions(+), 32 deletions(-)
diff --git a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java
index 3d0de0c..c4a81da 100644
--- a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java
+++ b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/utils/IgniteAwareApplicationService.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.ducktest.utils;
import java.util.Base64;
-import java.util.Optional;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.ignite.Ignite;
@@ -39,8 +38,17 @@ public class IgniteAwareApplicationService {
/** Logger. */
private static final Logger log = LogManager.getLogger(IgniteAwareApplicationService.class.getName());
- /** Thin client connection string variable. */
- public static final String THIN_CLIENT_CONNECTION = "thin_client_connection";
+ /** Application modes. */
+ private enum IgniteServiceType {
+ /** Server or client node. */
+ NODE,
+
+ /** Thin client connection. */
+ THIN_CLIENT,
+
+ /** Run application without precreated connections. */
+ NONE
+ }
/**
* @param args Args.
@@ -50,7 +58,7 @@ public class IgniteAwareApplicationService {
String[] params = args[0].split(",");
- boolean startIgnite = Boolean.parseBoolean(params[0]);
+ IgniteServiceType svcType = IgniteServiceType.valueOf(params[0]);
Class<?> clazz = Class.forName(params[1]);
@@ -61,16 +69,12 @@ public class IgniteAwareApplicationService {
JsonNode jsonNode = params.length > 3 ?
mapper.readTree(Base64.getDecoder().decode(params[3])) : mapper.createObjectNode();
- String tcConnStr = Optional.ofNullable(jsonNode.get(THIN_CLIENT_CONNECTION))
- .map(JsonNode::asText)
- .orElse(null);
-
IgniteAwareApplication app =
(IgniteAwareApplication)clazz.getConstructor().newInstance();
app.cfgPath = cfgPath;
- if (startIgnite) {
+ if (svcType == IgniteServiceType.NODE) {
log.info("Starting Ignite node...");
IgniteBiTuple<IgniteConfiguration, GridSpringResourceContext> cfgs = IgnitionEx.loadConfiguration(cfgPath);
@@ -86,8 +90,12 @@ public class IgniteAwareApplicationService {
log.info("Ignite instance closed. [interrupted=" + Thread.currentThread().isInterrupted() + "]");
}
}
- else if (tcConnStr != null && !tcConnStr.isEmpty()) {
- try (IgniteClient client = Ignition.startClient(new ClientConfiguration().setAddresses(tcConnStr))) {
+ else if (svcType == IgniteServiceType.THIN_CLIENT) {
+ log.info("Starting thin client...");
+
+ ClientConfiguration cfg = IgnitionEx.loadSpringBean(cfgPath, "thin.client.cfg");
+
+ try (IgniteClient client = Ignition.startClient(cfg)) {
app.client = client;
app.start(jsonNode);
@@ -95,7 +103,10 @@ public class IgniteAwareApplicationService {
finally {
log.info("Thin client instance closed. [interrupted=" + Thread.currentThread().isInterrupted() + "]");
}
- } else
+ }
+ else if (svcType == IgniteServiceType.NONE)
app.start(jsonNode);
+ else
+ throw new IllegalArgumentException("Unknown service type " + svcType);
}
}
diff --git a/modules/ducktests/tests/ignitetest/services/ignite_app.py b/modules/ducktests/tests/ignitetest/services/ignite_app.py
index e3c4885..59e87ff 100644
--- a/modules/ducktests/tests/ignitetest/services/ignite_app.py
+++ b/modules/ducktests/tests/ignitetest/services/ignite_app.py
@@ -38,10 +38,10 @@ class IgniteApplicationService(IgniteAwareService):
# pylint: disable=R0913
def __init__(self, context, config, java_class_name, num_nodes=1, params="", startup_timeout_sec=60,
shutdown_timeout_sec=10, modules=None, main_java_class=SERVICE_JAVA_CLASS_NAME, jvm_opts=None,
- full_jvm_opts=None, start_ignite=True):
+ full_jvm_opts=None):
super().__init__(context, config, num_nodes, startup_timeout_sec, shutdown_timeout_sec, main_java_class,
modules=modules, java_class_name=java_class_name, params=params,
- jvm_opts=jvm_opts, full_jvm_opts=full_jvm_opts, start_ignite=start_ignite)
+ jvm_opts=jvm_opts, full_jvm_opts=full_jvm_opts)
self.java_class_name = java_class_name
self.params = params
diff --git a/modules/ducktests/tests/ignitetest/services/utils/__init__.py b/modules/ducktests/tests/ignitetest/services/utils/__init__.py
index ec20143..c2830bc 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/__init__.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/__init__.py
@@ -12,3 +12,20 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+"""
+This module contains utility classes and enums.
+"""
+from enum import IntEnum
+
+from ignitetest.utils.enum import constructible
+
+
+@constructible
+class IgniteServiceType(IntEnum):
+ """
+ Application start mode.
+ """
+ NODE = 0
+ THIN_CLIENT = 1
+ NONE = 2
diff --git a/modules/ducktests/tests/ignitetest/services/utils/config_template.py b/modules/ducktests/tests/ignitetest/services/utils/config_template.py
index 4b1ff83..7ed35f9 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/config_template.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/config_template.py
@@ -22,6 +22,7 @@ from jinja2 import FileSystemLoader, Environment
DEFAULT_CONFIG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates")
DEFAULT_IGNITE_CONF = os.path.join(DEFAULT_CONFIG_PATH, "ignite.xml.j2")
+DEFAULT_THIN_CLIENT_CONF = os.path.join(DEFAULT_CONFIG_PATH, "thin_client_config.xml.j2")
class ConfigTemplate:
@@ -65,6 +66,14 @@ class IgniteClientConfigTemplate(ConfigTemplate):
self.default_params.update(client_mode=True)
+class IgniteThinClientConfigTemplate(ConfigTemplate):
+ """
+ Ignite client node configuration.
+ """
+ def __init__(self, path=DEFAULT_THIN_CLIENT_CONF):
+ super().__init__(path)
+
+
class IgniteLoggerConfigTemplate(ConfigTemplate):
"""
Ignite logger configuration.
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
index cc29379..ca906fd 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
@@ -29,6 +29,7 @@ from threading import Thread
from ducktape.cluster.remoteaccount import RemoteCommandError
from ducktape.utils.util import wait_until
+from ignitetest.services.utils import IgniteServiceType
from ignitetest.services.utils.background_thread import BackgroundThreadService
from ignitetest.services.utils.concurrent import CountDownLatch, AtomicValue
from ignitetest.services.utils.ignite_spec import resolve_spec
@@ -73,7 +74,6 @@ class IgniteAwareService(BackgroundThreadService, IgnitePathAware, metaclass=ABC
self.init_logs_attribute()
self.disconnected_nodes = []
- self.start_ignite = kwargs.get("start_ignite", True)
@property
def product(self):
@@ -97,7 +97,7 @@ class IgniteAwareService(BackgroundThreadService, IgnitePathAware, metaclass=ABC
"""
Awaits start finished.
"""
- if not self.start_ignite:
+ if self.config.service_type in (IgniteServiceType.NONE, IgniteServiceType.THIN_CLIENT):
return
self.logger.info("Waiting for IgniteAware(s) to start ...")
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py
index fe6179a..fa6efe92 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py
@@ -19,6 +19,7 @@ This module contains IgniteConfiguration classes and utilities.
import socket
from typing import NamedTuple
+from ignitetest.services.utils import IgniteServiceType
from ignitetest.services.utils.ignite_configuration.communication import CommunicationSpi, TcpCommunicationSpi
from ignitetest.services.utils.ssl.client_connector_configuration import ClientConnectorConfiguration
from ignitetest.services.utils.ssl.connector_configuration import ConnectorConfiguration
@@ -94,9 +95,38 @@ class IgniteConfiguration(NamedTuple):
"""
return self.__prepare_ssl(test_globals).__prepare_discovery(node, cluster)
+ @property
+ def service_type(self):
+ """
+ Application mode.
+ """
+ return IgniteServiceType.NODE
+
class IgniteClientConfiguration(IgniteConfiguration):
"""
Ignite client configuration.
"""
client_mode = True
+
+
+class IgniteThinClientConfiguration(NamedTuple):
+ """
+ Thin client configuration.
+ """
+ addresses: str = None
+ version: IgniteVersion = DEV_BRANCH
+
+ # pylint: disable=unused-argument
+ def prepare_for_env(self, test_globals, node, cluster):
+ """
+ Updates configuration based on current environment.
+ """
+ return self
+
+ @property
+ def service_type(self):
+ """
+ Application mode.
+ """
+ return IgniteServiceType.THIN_CLIENT
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py
index 8df5950..28f309c 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py
@@ -23,8 +23,9 @@ import json
import os
from abc import ABCMeta, abstractmethod
+from ignitetest.services.utils import IgniteServiceType
from ignitetest.services.utils.config_template import IgniteClientConfigTemplate, IgniteServerConfigTemplate, \
- IgniteLoggerConfigTemplate
+ IgniteLoggerConfigTemplate, IgniteThinClientConfigTemplate
from ignitetest.services.utils.jvm_utils import create_jvm_settings, merge_jvm_settings
from ignitetest.services.utils.path import get_home_dir, get_module_path, IgnitePathAware
from ignitetest.utils.version import DEV_BRANCH
@@ -84,11 +85,19 @@ class IgniteSpec(metaclass=ABCMeta):
"""
:return: config that service will use to start on a node
"""
- return [
- (IgnitePathAware.IGNITE_LOG_CONFIG_NAME, IgniteLoggerConfigTemplate()),
- (IgnitePathAware.IGNITE_CONFIG_NAME,
- IgniteClientConfigTemplate() if self.config.client_mode else IgniteServerConfigTemplate())
- ]
+ if self.config.service_type == IgniteServiceType.NONE:
+ return []
+
+ config_templates = [(IgnitePathAware.IGNITE_LOG_CONFIG_NAME, IgniteLoggerConfigTemplate())]
+
+ if self.config.service_type == IgniteServiceType.NODE:
+ config_templates.append((IgnitePathAware.IGNITE_CONFIG_NAME,
+ IgniteClientConfigTemplate() if self.config.client_mode
+ else IgniteServerConfigTemplate()))
+ else:
+ config_templates.append((IgnitePathAware.IGNITE_THIN_CLIENT_CONFIG_NAME, IgniteThinClientConfigTemplate()))
+
+ return config_templates
def __home(self, product=None):
"""
@@ -198,7 +207,7 @@ class ApacheIgniteApplicationSpec(IgniteApplicationSpec):
Implementation IgniteApplicationSpec for Apache Ignite project
"""
# pylint: disable=too-many-arguments
- def __init__(self, context, modules, main_java_class, java_class_name, params, start_ignite, **kwargs):
+ def __init__(self, context, modules, main_java_class, java_class_name, params, **kwargs):
super().__init__(**kwargs)
self.context = context
@@ -224,10 +233,13 @@ class ApacheIgniteApplicationSpec(IgniteApplicationSpec):
"-ea",
"-DIGNITE_ALLOW_ATOMIC_OPS_IN_TX=false"])
+ config_file = self.path_aware.config_file if self.config.service_type == IgniteServiceType.NODE \
+ else self.path_aware.thin_client_config_file
+
self.args = [
- str(start_ignite),
+ str(self.config.service_type.name),
java_class_name,
- self.path_aware.config_file,
+ config_file,
str(base64.b64encode(json.dumps(params).encode('utf-8')), 'utf-8')
]
diff --git a/modules/ducktests/tests/ignitetest/services/utils/path.py b/modules/ducktests/tests/ignitetest/services/utils/path.py
index 7e559b1..358ecfe 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/path.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/path.py
@@ -153,14 +153,29 @@ class IgnitePathAware(PathAware, metaclass=ABCMeta):
"""
IGNITE_CONFIG_NAME = "ignite-config.xml"
+ IGNITE_THIN_CLIENT_CONFIG_NAME = "ignite-thin-config.xml"
+
IGNITE_LOG_CONFIG_NAME = "ignite-log4j.xml"
@property
def config_file(self):
+ """
+ :return: path to config file
+ """
return os.path.join(self.config_dir, IgnitePathAware.IGNITE_CONFIG_NAME)
@property
+ def thin_client_config_file(self):
+ """
+ :return: path to thin client config file
+ """
+ return os.path.join(self.config_dir, IgnitePathAware.IGNITE_THIN_CLIENT_CONFIG_NAME)
+
+ @property
def log_config_file(self):
+ """
+ :return: path to log config file
+ """
return os.path.join(self.config_dir, IgnitePathAware.IGNITE_LOG_CONFIG_NAME)
@property
diff --git a/modules/ducktests/tests/ignitetest/services/utils/templates/thin_client_config.xml.j2 b/modules/ducktests/tests/ignitetest/services/utils/templates/thin_client_config.xml.j2
new file mode 100644
index 0000000..14ec921
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/services/utils/templates/thin_client_config.xml.j2
@@ -0,0 +1,29 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+#}
+
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
+ <bean class="org.apache.ignite.configuration.ClientConfiguration" id="thin.client.cfg">
+ <property name="addresses">
+ <list>
+ <value>{{ config.addresses }}</value>
+ </list>
+ </property>
+ </bean>
+</beans>
diff --git a/modules/ducktests/tests/ignitetest/tests/thin_client_test.py b/modules/ducktests/tests/ignitetest/tests/thin_client_test.py
index 9f7fe23..d7e3eae 100644
--- a/modules/ducktests/tests/ignitetest/tests/thin_client_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/thin_client_test.py
@@ -20,7 +20,7 @@ from ducktape.mark import matrix
from ignitetest.services.ignite import IgniteService
from ignitetest.services.ignite_app import IgniteApplicationService
-from ignitetest.services.utils.ignite_configuration import IgniteConfiguration
+from ignitetest.services.utils.ignite_configuration import IgniteConfiguration, IgniteThinClientConfiguration
from ignitetest.services.utils.ssl.client_connector_configuration import ClientConnectorConfiguration
from ignitetest.utils import cluster
from ignitetest.utils.ignite_test import IgniteTest
@@ -48,15 +48,14 @@ class ThinClientTest(IgniteTest):
ignite = IgniteService(self.test_context, server_config, 1)
- thin_client_connection = ignite.nodes[0].account.hostname + ":" + str(server_config.
- client_connector_configuration.port)
+ addresses = ignite.nodes[0].account.hostname + ":" + str(server_config.client_connector_configuration.port)
thin_clients = IgniteApplicationService(self.test_context,
- IgniteConfiguration(version=IgniteVersion(thin_client_version)),
+ IgniteThinClientConfiguration(addresses=addresses,
+ version=IgniteVersion(
+ thin_client_version)),
java_class_name=self.JAVA_CLIENT_CLASS_NAME,
- num_nodes=1,
- params={"thin_client_connection": thin_client_connection},
- start_ignite=False)
+ num_nodes=1)
ignite.start()
thin_clients.run()