You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by "github-code-scanning[bot] (via GitHub)" <gi...@apache.org> on 2023/02/05 06:37:04 UTC
[GitHub] [druid] github-code-scanning[bot] commented on a diff in pull request #13669: Generate the IT docker-compose.yaml files
github-code-scanning[bot] commented on code in PR #13669:
URL: https://github.com/apache/druid/pull/13669#discussion_r1096631151
##########
integration-tests-ex/cases/templates/template.py:
##########
@@ -0,0 +1,314 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Generates a docker-compose.yaml file from a test-specific template. Each
+test template either uses the base template directly, or extends the template
+to customize bits of the cluster.
+
+Since the cluster is defined as YAML, the cluster definition is build up
+internally as a Python data structure made up of maps, arrays and scalars.
+PyYaml does the grunt work of converting the data structure to the YAML file.
+'''
+
+import yaml, os, sys, os.path
+from pathlib import Path
+
+DRUID_NETWORK = 'druid-it-net'
+DRUID_SUBNET = '172.172.172'
+ZOO_KEEPER = 'zookeeper'
+METADATA = 'metadata'
+COORDINATOR = 'coordinator'
+OVERLORD = 'overlord'
+ROUTER = 'router'
+BROKER = 'broker'
+HISTORICAL = 'historical'
+INDEXER = 'indexer'
+MIDDLE_MANAGER = 'middlemanager'
+
+def generate(template_path, template):
+ #template_path = Path(sys.modules[template.__class__.__module__].__file__)
+ template_path = Path(template_path)
+ #print("template_path", template_path)
+ cluster = template_path.stem
+ #print("Cluster", cluster)
+ module_dir = Path(__file__).parent.parent
+ target_dir = module_dir.joinpath("target")
+ target_file = target_dir.joinpath('cluster', cluster, 'docker-compose.yaml')
+ #print("target_file", target_file)
+ with target_file.open("w") as f:
+ template.generate_file(f, cluster)
+ f.close()
+
+class BaseTemplate:
+
+ def __init__(self):
+ self.cluster = {}
+
+ def generate_file(self, out_file, cluster):
+ self.cluster_name = cluster
+ self.define_cluster()
+ self.out_file = out_file
+ self.generate()
+
+ def define_cluster(self):
+ self.define_network()
+ self.define_support_services()
+ self.define_druid_services()
+ self.define_custom_services()
+
+ def define_support_services(self):
+ self.define_zk()
+ self.define_metadata()
+
+ def define_druid_services(self):
+ self.define_coordinator()
+ self.define_overlord()
+ self.define_broker()
+ self.define_router()
+ self.define_historical()
+ self.define_indexer()
+
+ def define_custom_services(self):
+ '''
+ Override to define additional services for the cluster.
+ '''
+ pass
+
+ def generate(self):
+ self.gen_header()
+ self.gen_header_comment()
+ self.gen_body()
+
+ def emit(self, text):
+ # Chop off the newline that occurs when ''' is on a separate line
+ if len(text) > 0 and text[0] == '\n':
+ text = text[1:]
+ self.out_file.write(text)
+
+ def gen_header(self):
+ self.emit('''
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#--------------------------------------------------------------------
+
+# THIS FILE IS GENERATED -- DO NOT EDIT!
+#
+# Instead, edit the template from which this file was generated.
+# Template: templates/{}.py
+
+'''.format(self.cluster_name))
+
+ def gen_header_comment(self):
+ '''
+ Override to generate a custom header comment after the standard header.
+ '''
+ pass
+
+ def gen_body(self):
+ yaml.dump(self.cluster, self.out_file, sort_keys=False)
+
+ def define_network(self):
+ self.cluster['networks'] = {
+ 'druid-it-net': {
+ 'name': DRUID_NETWORK,
+ 'ipam': {
+ 'config': [
+ {'subnet': DRUID_SUBNET + '.0/24'}
+ ]
+ }
+ }
+ }
+
+ def add_service(self, name, service):
+ services = self.cluster.setdefault('services', {})
+ services[name] = service
+
+ def add_volume(self, service, local, container):
+ '''
+ Add a volume to a service.
+ '''
+ volumes = service.setdefault('volumes', [])
+ volumes.append(local + ':' + container)
+
+ def add_env(self, service, var, value):
+ '''
+ Add an environment variable to a service.
+ '''
+ vars = service.setdefault('environment', [])
+ vars.append(var + '=' + value)
+
+ def add_property(self, service, prop, value):
+ '''
+ Sets a property for a service. The property is of the same form as the
+ .properties file: druid.some.property.
+ This method converts the property to the env var form so you don't have to.
+ '''
+ var = prop.replace('.', '_')
+ self.add_env(service, var, value)
+
+ def add_env_file(self, service, env_file):
+ '''
+ Add an environment file to a service.
+ '''
+ env_files = service.setdefault('env_file', [])
+ env_files.append(env_file)
+
+ def add_env_config(self, service, base_name):
+ '''
+ Add to a service one of the standard environment config files in
+ the Common/environment-configs directory
+ '''
+ self.add_env_file(service, '../Common/environment-configs/' + base_name + '.env')
+
+ def add_port(self, service, local, container):
+ '''
+ Add a port mapping to the service
+ '''
+ ports = service.setdefault('ports', [])
+ ports.append(local + ':' + container)
+
+ def define_external_service(self, name) -> dict:
+ service = {'extends': {
+ 'file': '../Common/dependencies.yaml',
+ 'service': name
+ }}
+ self.add_service(name, service)
+ return service
+
+ def define_zk(self):
+ self.define_external_service(ZOO_KEEPER)
+
+ def define_metadata(self):
+ self.define_external_service(METADATA)
+
+ def define_druid_service(self, name, base):
+ service = {}
+ if base is not None:
+ service['extends'] = {
+ 'file': '../Common/druid.yaml',
+ 'service': base
+ }
+ self.extend_druid_service(service)
+ self.add_service(name, service)
+ return service
+
+ def extend_druid_service(self, service):
+ '''
+ Override this to add options to all Druid services.
+ '''
+ pass
+
+ def add_depends(self, service, items):
+ if items is None or len(items) == 0:
+ return
+ depends = service.setdefault('depends_on', [])
+ depends += items
+ return depends
+
+ def define_master_service(self, name, base):
+ service = self.define_druid_service(name, base)
+ self.add_depends(service, [ZOO_KEEPER, METADATA])
+ return service
+
+ def define_std_master_service(self, name):
+ return self.define_master_service(name, name)
+
+ def define_coordinator(self):
+ return self.define_std_master_service(COORDINATOR)
+ return service
Review Comment:
## Unreachable code
This statement is unreachable.
[Show more details](https://github.com/apache/druid/security/code-scanning/4232)
##########
integration-tests-ex/cases/templates/template.py:
##########
@@ -0,0 +1,314 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Generates a docker-compose.yaml file from a test-specific template. Each
+test template either uses the base template directly, or extends the template
+to customize bits of the cluster.
+
+Since the cluster is defined as YAML, the cluster definition is build up
+internally as a Python data structure made up of maps, arrays and scalars.
+PyYaml does the grunt work of converting the data structure to the YAML file.
+'''
+
+import yaml, os, sys, os.path
Review Comment:
## Unused import
Import of 'sys' is not used.
[Show more details](https://github.com/apache/druid/security/code-scanning/4231)
##########
integration-tests-ex/cases/templates/template.py:
##########
@@ -0,0 +1,314 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Generates a docker-compose.yaml file from a test-specific template. Each
+test template either uses the base template directly, or extends the template
+to customize bits of the cluster.
+
+Since the cluster is defined as YAML, the cluster definition is build up
+internally as a Python data structure made up of maps, arrays and scalars.
+PyYaml does the grunt work of converting the data structure to the YAML file.
+'''
+
+import yaml, os, sys, os.path
+from pathlib import Path
+
+DRUID_NETWORK = 'druid-it-net'
+DRUID_SUBNET = '172.172.172'
+ZOO_KEEPER = 'zookeeper'
+METADATA = 'metadata'
+COORDINATOR = 'coordinator'
+OVERLORD = 'overlord'
+ROUTER = 'router'
+BROKER = 'broker'
+HISTORICAL = 'historical'
+INDEXER = 'indexer'
+MIDDLE_MANAGER = 'middlemanager'
+
+def generate(template_path, template):
+ #template_path = Path(sys.modules[template.__class__.__module__].__file__)
+ template_path = Path(template_path)
+ #print("template_path", template_path)
+ cluster = template_path.stem
+ #print("Cluster", cluster)
+ module_dir = Path(__file__).parent.parent
+ target_dir = module_dir.joinpath("target")
+ target_file = target_dir.joinpath('cluster', cluster, 'docker-compose.yaml')
+ #print("target_file", target_file)
+ with target_file.open("w") as f:
+ template.generate_file(f, cluster)
+ f.close()
+
+class BaseTemplate:
+
+ def __init__(self):
+ self.cluster = {}
+
+ def generate_file(self, out_file, cluster):
+ self.cluster_name = cluster
+ self.define_cluster()
+ self.out_file = out_file
+ self.generate()
+
+ def define_cluster(self):
+ self.define_network()
+ self.define_support_services()
+ self.define_druid_services()
+ self.define_custom_services()
+
+ def define_support_services(self):
+ self.define_zk()
+ self.define_metadata()
+
+ def define_druid_services(self):
+ self.define_coordinator()
+ self.define_overlord()
+ self.define_broker()
+ self.define_router()
+ self.define_historical()
+ self.define_indexer()
+
+ def define_custom_services(self):
+ '''
+ Override to define additional services for the cluster.
+ '''
+ pass
+
+ def generate(self):
+ self.gen_header()
+ self.gen_header_comment()
+ self.gen_body()
+
+ def emit(self, text):
+ # Chop off the newline that occurs when ''' is on a separate line
+ if len(text) > 0 and text[0] == '\n':
+ text = text[1:]
+ self.out_file.write(text)
+
+ def gen_header(self):
+ self.emit('''
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#--------------------------------------------------------------------
+
+# THIS FILE IS GENERATED -- DO NOT EDIT!
+#
+# Instead, edit the template from which this file was generated.
+# Template: templates/{}.py
+
+'''.format(self.cluster_name))
+
+ def gen_header_comment(self):
+ '''
+ Override to generate a custom header comment after the standard header.
+ '''
+ pass
+
+ def gen_body(self):
+ yaml.dump(self.cluster, self.out_file, sort_keys=False)
+
+ def define_network(self):
+ self.cluster['networks'] = {
+ 'druid-it-net': {
+ 'name': DRUID_NETWORK,
+ 'ipam': {
+ 'config': [
+ {'subnet': DRUID_SUBNET + '.0/24'}
+ ]
+ }
+ }
+ }
+
+ def add_service(self, name, service):
+ services = self.cluster.setdefault('services', {})
+ services[name] = service
+
+ def add_volume(self, service, local, container):
+ '''
+ Add a volume to a service.
+ '''
+ volumes = service.setdefault('volumes', [])
+ volumes.append(local + ':' + container)
+
+ def add_env(self, service, var, value):
+ '''
+ Add an environment variable to a service.
+ '''
+ vars = service.setdefault('environment', [])
+ vars.append(var + '=' + value)
+
+ def add_property(self, service, prop, value):
+ '''
+ Sets a property for a service. The property is of the same form as the
+ .properties file: druid.some.property.
+ This method converts the property to the env var form so you don't have to.
+ '''
+ var = prop.replace('.', '_')
+ self.add_env(service, var, value)
+
+ def add_env_file(self, service, env_file):
+ '''
+ Add an environment file to a service.
+ '''
+ env_files = service.setdefault('env_file', [])
+ env_files.append(env_file)
+
+ def add_env_config(self, service, base_name):
+ '''
+ Add to a service one of the standard environment config files in
+ the Common/environment-configs directory
+ '''
+ self.add_env_file(service, '../Common/environment-configs/' + base_name + '.env')
+
+ def add_port(self, service, local, container):
+ '''
+ Add a port mapping to the service
+ '''
+ ports = service.setdefault('ports', [])
+ ports.append(local + ':' + container)
+
+ def define_external_service(self, name) -> dict:
+ service = {'extends': {
+ 'file': '../Common/dependencies.yaml',
+ 'service': name
+ }}
+ self.add_service(name, service)
+ return service
+
+ def define_zk(self):
+ self.define_external_service(ZOO_KEEPER)
+
+ def define_metadata(self):
+ self.define_external_service(METADATA)
+
+ def define_druid_service(self, name, base):
+ service = {}
+ if base is not None:
+ service['extends'] = {
+ 'file': '../Common/druid.yaml',
+ 'service': base
+ }
+ self.extend_druid_service(service)
+ self.add_service(name, service)
+ return service
+
+ def extend_druid_service(self, service):
+ '''
+ Override this to add options to all Druid services.
+ '''
+ pass
+
+ def add_depends(self, service, items):
Review Comment:
## Explicit returns mixed with implicit (fall through) returns
Mixing implicit and explicit returns may indicate an error as implicit returns always return None.
[Show more details](https://github.com/apache/druid/security/code-scanning/4230)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org