You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by "abhishekagarwal87 (via GitHub)" <gi...@apache.org> on 2023/02/15 10:08:08 UTC
[GitHub] [druid] abhishekagarwal87 commented on a diff in pull request #13669: Generate the IT docker-compose.yaml files
abhishekagarwal87 commented on code in PR #13669:
URL: https://github.com/apache/druid/pull/13669#discussion_r1106905070
##########
integration-tests-ex/cases/cluster.sh:
##########
@@ -54,114 +62,159 @@ shift
# All commands need env vars
ENV_FILE=$MODULE_DIR/../image/target/env.sh
if [ ! -f $ENV_FILE ]; then
- echo "Please build the Docker test image before testing" 1>&2
- exit 1
+ echo "Please build the Docker test image before testing" 1>&2
+ exit 1
fi
source $ENV_FILE
function category {
- if [ $# -eq 0 ]; then
- usage 1>&2
- exit 1
- fi
- export CATEGORY=$1
- # The untranslated category is used for the local name of the
- # shared folder.
-
- # DRUID_INTEGRATION_TEST_GROUP is used in
- # docker-compose files and here. Despite the name, it is the
- # name of the cluster configuration we want to run, not the
- # test category. Multiple categories can map to the same cluster
- # definition.
-
- # Map from category name to shared cluster definition name.
- # Add an entry here if you create a new category that shares
- # a definition.
- case $CATEGORY in
- "InputSource")
- export DRUID_INTEGRATION_TEST_GROUP=BatchIndex
- ;;
- "InputFormat")
- export DRUID_INTEGRATION_TEST_GROUP=BatchIndex
- ;;
- "Catalog")
- export DRUID_INTEGRATION_TEST_GROUP=BatchIndex
- ;;
- *)
- export DRUID_INTEGRATION_TEST_GROUP=$CATEGORY
- ;;
- esac
-
- export CLUSTER_DIR=$MODULE_DIR/cluster/$DRUID_INTEGRATION_TEST_GROUP
- if [ ! -d $CLUSTER_DIR ]; then
- echo "Cluster directory $CLUSTER_DIR does not exist." 1>&2
- echo "$USAGE" 1>&2
- exit 1
- fi
-
- export TARGET_DIR=$MODULE_DIR/target
- export SHARED_DIR=$TARGET_DIR/$CATEGORY
- export ENV_FILE="$TARGET_DIR/${CATEGORY}.env"
+ if [ $# -eq 0 ]; then
+ usage 1>&2
+ exit 1
+ fi
+ export CATEGORY=$1
+ # The untranslated category is used for the local name of the
+ # shared folder.
+
+ # DRUID_INTEGRATION_TEST_GROUP is used in
+ # docker-compose files and here. Despite the name, it is the
+ # name of the cluster configuration we want to run, not the
+ # test category. Multiple categories can map to the same cluster
+ # definition.
+
+ # Map from category name to shared cluster definition name.
+ # Add an entry here if you create a new category that shares
+ # a definition.
+ case $CATEGORY in
+ "InputSource")
+ export DRUID_INTEGRATION_TEST_GROUP=BatchIndex
+ ;;
+ "InputFormat")
+ export DRUID_INTEGRATION_TEST_GROUP=BatchIndex
+ ;;
+ "Catalog")
+ export DRUID_INTEGRATION_TEST_GROUP=BatchIndex
+ ;;
+ *)
+ export DRUID_INTEGRATION_TEST_GROUP=$CATEGORY
+ ;;
+ esac
+
+ export CLUSTER_DIR=$MODULE_DIR/cluster/$DRUID_INTEGRATION_TEST_GROUP
+ export TARGET_DIR=$MODULE_DIR/target
+ export SHARED_DIR=$TARGET_DIR/$CATEGORY
+ export ENV_FILE="$TARGET_DIR/${CATEGORY}.env"
}
# Dump lots of information to debug Docker failures when run inside
# of a build environment where we can't inspect Docker directly.
function show_status {
- echo "===================================="
- ls -l target/shared
- echo "docker ps -a"
- docker ps -a
- # Was: --filter status=exited
- for id in $(docker ps -a --format "{{.ID}}"); do
- echo "===================================="
- echo "Logs for Container ID $id"
- docker logs $id | tail -n 20
- done
- echo "===================================="
+ echo "===================================="
+ ls -l target/shared
+ echo "docker ps -a"
+ docker ps -a
+ # Was: --filter status=exited
+ for id in $(docker ps -a --format "{{.ID}}"); do
+ echo "===================================="
+ echo "Logs for Container ID $id"
+ docker logs $id | tail -n 20
+ done
+ echo "===================================="
}
function build_shared_dir {
- mkdir -p $SHARED_DIR
- # Must start with an empty DB to keep MySQL happy
- rm -rf $SHARED_DIR/db
- mkdir -p $SHARED_DIR/logs
- mkdir -p $SHARED_DIR/tasklogs
- mkdir -p $SHARED_DIR/db
- mkdir -p $SHARED_DIR/kafka
- mkdir -p $SHARED_DIR/resources
- cp $MODULE_DIR/assets/log4j2.xml $SHARED_DIR/resources
- # Permissions in some build setups are screwed up. See above. The user
- # which runs Docker does not have permission to write into the /shared
- # directory. Force ownership to allow writing.
- chmod -R a+rwx $SHARED_DIR
+ mkdir -p $SHARED_DIR
+ # Must start with an empty DB to keep MySQL happy
+ rm -rf $SHARED_DIR/db
+ mkdir -p $SHARED_DIR/logs
+ mkdir -p $SHARED_DIR/tasklogs
+ mkdir -p $SHARED_DIR/db
+ mkdir -p $SHARED_DIR/kafka
+ mkdir -p $SHARED_DIR/resources
+ cp $MODULE_DIR/assets/log4j2.xml $SHARED_DIR/resources
+ # Permissions in some build setups are screwed up. See above. The user
+ # which runs Docker does not have permission to write into the /shared
+ # directory. Force ownership to allow writing.
+ chmod -R a+rwx $SHARED_DIR
+}
+
+# Either generate the docker-compose file, or use "static" versions.
+function docker_file {
+
+ # If a template exists, generate the docker-compose.yaml file. Copy over the Common
+ # folder.
+ TEMPLATE_DIR=$MODULE_DIR/templates
+ TEMPLATE_FILE=${DRUID_INTEGRATION_TEST_GROUP}.py
Review Comment:
the name `template_file` seems confusing as it is not really a template.
##########
integration-tests-ex/cases/templates/template.py:
##########
@@ -0,0 +1,319 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Generates a docker-compose.yaml file from a test-specific template. Each
+test template either uses the base template directly, or extends the template
+to customize bits of the cluster.
+
+Since the cluster is defined as YAML, the cluster definition is build up
+internally as a Python data structure made up of maps, arrays and scalars.
+PyYaml does the grunt work of converting the data structure to the YAML file.
+'''
+
+import yaml, os, os.path
+from pathlib import Path
+
+DRUID_NETWORK = 'druid-it-net'
+DRUID_SUBNET = '172.172.172'
+ZOO_KEEPER = 'zookeeper'
+METADATA = 'metadata'
+COORDINATOR = 'coordinator'
+OVERLORD = 'overlord'
+ROUTER = 'router'
+BROKER = 'broker'
+HISTORICAL = 'historical'
+INDEXER = 'indexer'
+MIDDLE_MANAGER = 'middlemanager'
+
+def generate(template_path, template):
+ #template_path = Path(sys.modules[template.__class__.__module__].__file__)
+ template_path = Path(template_path)
+ #print("template_path", template_path)
+ cluster = template_path.stem
+ #print("Cluster", cluster)
+ module_dir = Path(__file__).parent.parent
+ target_dir = module_dir.joinpath("target")
+ target_file = target_dir.joinpath('cluster', cluster, 'docker-compose.yaml')
+ #print("target_file", target_file)
+ with target_file.open("w") as f:
+ template.generate_file(f, cluster)
+ f.close()
Review Comment:
can you please remove the commented code?
##########
integration-tests-ex/cases/templates/template.py:
##########
@@ -0,0 +1,319 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Generates a docker-compose.yaml file from a test-specific template. Each
+test template either uses the base template directly, or extends the template
+to customize bits of the cluster.
+
+Since the cluster is defined as YAML, the cluster definition is build up
+internally as a Python data structure made up of maps, arrays and scalars.
+PyYaml does the grunt work of converting the data structure to the YAML file.
+'''
+
+import yaml, os, os.path
+from pathlib import Path
+
+DRUID_NETWORK = 'druid-it-net'
+DRUID_SUBNET = '172.172.172'
+ZOO_KEEPER = 'zookeeper'
+METADATA = 'metadata'
+COORDINATOR = 'coordinator'
+OVERLORD = 'overlord'
+ROUTER = 'router'
+BROKER = 'broker'
+HISTORICAL = 'historical'
+INDEXER = 'indexer'
+MIDDLE_MANAGER = 'middlemanager'
+
+def generate(template_path, template):
+ #template_path = Path(sys.modules[template.__class__.__module__].__file__)
+ template_path = Path(template_path)
+ #print("template_path", template_path)
+ cluster = template_path.stem
+ #print("Cluster", cluster)
+ module_dir = Path(__file__).parent.parent
+ target_dir = module_dir.joinpath("target")
+ target_file = target_dir.joinpath('cluster', cluster, 'docker-compose.yaml')
+ #print("target_file", target_file)
+ with target_file.open("w") as f:
+ template.generate_file(f, cluster)
+ f.close()
+
+class BaseTemplate:
+
+ def __init__(self):
+ self.cluster = {}
+
+ def generate_file(self, out_file, cluster):
+ self.cluster_name = cluster
+ self.define_cluster()
+ self.out_file = out_file
+ self.generate()
+
+ def define_cluster(self):
+ self.define_network()
+ self.define_support_services()
+ self.define_druid_services()
+ self.define_custom_services()
+
+ def define_support_services(self):
+ self.define_zk()
+ self.define_metadata()
+
+ def define_druid_services(self):
+ self.define_coordinator()
+ self.define_overlord()
+ self.define_broker()
+ self.define_router()
+ self.define_historical()
+ self.define_indexer()
+
+ def define_custom_services(self):
+ '''
+ Override to define additional services for the cluster.
+ '''
+ pass
+
+ def generate(self):
+ self.gen_header()
+ self.gen_header_comment()
+ self.gen_body()
+
+ def emit(self, text):
+ # Chop off the newline that occurs when ''' is on a separate line
+ if len(text) > 0 and text[0] == '\n':
+ text = text[1:]
+ self.out_file.write(text)
+
+ def gen_header(self):
+ self.emit('''
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#--------------------------------------------------------------------
Review Comment:
Is this header needed if we are not checking in the generated files?
##########
integration-tests-ex/cases/templates/template.py:
##########
@@ -0,0 +1,319 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Generates a docker-compose.yaml file from a test-specific template. Each
+test template either uses the base template directly, or extends the template
+to customize bits of the cluster.
+
+Since the cluster is defined as YAML, the cluster definition is build up
+internally as a Python data structure made up of maps, arrays and scalars.
+PyYaml does the grunt work of converting the data structure to the YAML file.
+'''
+
+import yaml, os, os.path
+from pathlib import Path
+
+DRUID_NETWORK = 'druid-it-net'
+DRUID_SUBNET = '172.172.172'
+ZOO_KEEPER = 'zookeeper'
+METADATA = 'metadata'
+COORDINATOR = 'coordinator'
+OVERLORD = 'overlord'
+ROUTER = 'router'
+BROKER = 'broker'
+HISTORICAL = 'historical'
+INDEXER = 'indexer'
+MIDDLE_MANAGER = 'middlemanager'
+
+def generate(template_path, template):
+ #template_path = Path(sys.modules[template.__class__.__module__].__file__)
+ template_path = Path(template_path)
+ #print("template_path", template_path)
+ cluster = template_path.stem
+ #print("Cluster", cluster)
+ module_dir = Path(__file__).parent.parent
+ target_dir = module_dir.joinpath("target")
+ target_file = target_dir.joinpath('cluster', cluster, 'docker-compose.yaml')
+ #print("target_file", target_file)
+ with target_file.open("w") as f:
+ template.generate_file(f, cluster)
+ f.close()
Review Comment:
you can though add examples in the comments for target_dir. Not necessary, but will be nice to have.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org