You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2023/11/10 17:31:16 UTC
(lucene) branch main updated: script to run microbenchmarks across different ec2 instance types (#12787)
This is an automated email from the ASF dual-hosted git repository.
rmuir pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new c28d174cd7c script to run microbenchmarks across different ec2 instance types (#12787)
c28d174cd7c is described below
commit c28d174cd7ca71de962ee23d1891f0cacca8ec44
Author: Robert Muir <rm...@apache.org>
AuthorDate: Fri Nov 10 12:31:10 2023 -0500
script to run microbenchmarks across different ec2 instance types (#12787)
---
dev-tools/aws-jmh/Makefile | 45 ++++++++++
dev-tools/aws-jmh/README.txt | 13 +++
dev-tools/aws-jmh/cloud-init.yml.j2 | 8 ++
dev-tools/aws-jmh/cloudformation.yml.j2 | 49 ++++++++++
dev-tools/aws-jmh/destroy.yml | 9 ++
dev-tools/aws-jmh/group_vars/all.yml | 65 ++++++++++++++
dev-tools/aws-jmh/playbook.yml | 155 ++++++++++++++++++++++++++++++++
dev-tools/aws-jmh/requirements.txt | 4 +
gradle/validation/rat-sources.gradle | 3 +
9 files changed, 351 insertions(+)
diff --git a/dev-tools/aws-jmh/Makefile b/dev-tools/aws-jmh/Makefile
new file mode 100644
index 00000000000..e1e761b9357
--- /dev/null
+++ b/dev-tools/aws-jmh/Makefile
@@ -0,0 +1,45 @@
+.PHONY: test clean env
+.DELETE_ON_ERROR:
+
+# python interpreter to use
+export PYTHON=python3
+# improve ansible output
+export ANSIBLE_CALLBACK_RESULT_FORMAT=yaml
+# reasonable default verbosity, crank to debug issues
+export ANSIBLE_VERBOSITY=0
+# pipeline operations over ssh
+export ANSIBLE_PIPELINING=true
+# show breakdown of task times
+export ANSIBLE_CALLBACKS_ENABLED=profile_tasks
+# not useful since they are throwaways
+export ANSIBLE_HOST_KEY_CHECKING=false
+# run all machines in parallel
+export ANSIBLE_FORKS=10
+
+ifeq ($(AWS_ACCESS_KEY_ID),)
+ $(error AWS_ACCESS_KEY_ID is not set)
+endif
+
+ifeq ($(AWS_SECRET_ACCESS_KEY),)
+ $(error AWS_SECRET_ACCESS_KEY is not set)
+endif
+
+ifeq ($(PATCH_BRANCH),)
+ $(error PATCH_BRANCH is not set)
+endif
+
+# runs benchmark across AWS instances
+# always destroys anything created out there
+test: env
+ bash -c "set -ex; trap 'build/.env/bin/ansible-playbook destroy.yml' EXIT; build/.env/bin/ansible-playbook playbook.yml"
+
+clean:
+ rm -rf build
+
+# keep venv in sync with requirements.txt
+env: build/.env/bin/activate
+build/.env/bin/activate: requirements.txt
+ rm -rf build/.env
+ $(PYTHON) -m venv build/.env
+ build/.env/bin/pip install --no-input -U -r requirements.txt
+ touch build/.env/bin/activate
diff --git a/dev-tools/aws-jmh/README.txt b/dev-tools/aws-jmh/README.txt
new file mode 100644
index 00000000000..ee2d4bb322f
--- /dev/null
+++ b/dev-tools/aws-jmh/README.txt
@@ -0,0 +1,13 @@
+Runs lucene microbenchmarks across a variety of CPUs in EC2.
+
+Example:
+
+export AWS_ACCESS_KEY_ID=xxxxx
+export AWS_SECRET_ACCESS_KEY=yyyy
+make PATCH_BRANCH=rmuir:some-speedup
+
+Results file will be in build/report.txt
+
+You can also pass additional JMH args if you want:
+
+make PATCH_BRANCH=rmuir:some-speedup JMH_ARGS='float -p size=756'
diff --git a/dev-tools/aws-jmh/cloud-init.yml.j2 b/dev-tools/aws-jmh/cloud-init.yml.j2
new file mode 100644
index 00000000000..4463d79ef2b
--- /dev/null
+++ b/dev-tools/aws-jmh/cloud-init.yml.j2
@@ -0,0 +1,8 @@
+#cloud-config
+package_upgrade: false
+package_update: false
+system_info:
+ default_user:
+ name: {{ user }}
+ ssh_authorized_keys:
+ - {{ lookup('file', ssh_key + '.pub') }}
diff --git a/dev-tools/aws-jmh/cloudformation.yml.j2 b/dev-tools/aws-jmh/cloudformation.yml.j2
new file mode 100644
index 00000000000..82ac8e55978
--- /dev/null
+++ b/dev-tools/aws-jmh/cloudformation.yml.j2
@@ -0,0 +1,49 @@
+AWSTemplateFormatVersion: 2010-09-09
+Resources:
+
+ SecurityGroupIngress:
+ Type: AWS::EC2::SecurityGroup
+ Properties:
+ VpcId: {{ vpc_info.vpcs[0].id }}
+ GroupDescription: Allow inbound SSH traffic
+ SecurityGroupIngress:
+ - IpProtocol: "tcp"
+ FromPort: 22
+ ToPort: 22
+ CidrIp: "0.0.0.0/0"
+ Tags:
+ - Key: Name
+ Value: {{ stack_name }}-inbound-ssh
+
+ SecurityGroupEgress:
+ Type: AWS::EC2::SecurityGroup
+ Properties:
+ VpcId: {{ vpc_info.vpcs[0].id }}
+ GroupDescription: Allow all outbound traffic
+ SecurityGroupEgress:
+ - IpProtocol: "-1"
+ CidrIp: "0.0.0.0/0"
+ Tags:
+ - Key: Name
+ Value: {{ stack_name }}-outbound-all
+
+{% for instance in instances %}
+ {{ instance.name }}Instance:
+ Type: AWS::EC2::Instance
+ DependsOn:
+ - SecurityGroupIngress
+ - SecurityGroupEgress
+ Properties:
+ InstanceType: {{ instance.type }}
+ InstanceInitiatedShutdownBehavior: terminate
+ ImageId: {{ instance.ami }}
+ SecurityGroupIds:
+ - !Ref SecurityGroupIngress
+ - !Ref SecurityGroupEgress
+ UserData: {{ lookup('template', 'cloud-init.yml.j2') | b64encode }}
+ Tags:
+ - Key: Name
+ Value: {{ stack_name }}-{{ instance.name }}
+ - Key: ShortName
+ Value: {{ instance.name }}
+{% endfor %}
diff --git a/dev-tools/aws-jmh/destroy.yml b/dev-tools/aws-jmh/destroy.yml
new file mode 100644
index 00000000000..38beda2276c
--- /dev/null
+++ b/dev-tools/aws-jmh/destroy.yml
@@ -0,0 +1,9 @@
+- hosts: localhost
+ gather_facts: false
+ tasks:
+ - name: Destroy cloudformation stack
+ cloudformation:
+ stack_name: "{{ stack_name }}"
+ region: "{{ region }}"
+ state: absent
+ delegate_to: localhost
diff --git a/dev-tools/aws-jmh/group_vars/all.yml b/dev-tools/aws-jmh/group_vars/all.yml
new file mode 100644
index 00000000000..23c14c70b1e
--- /dev/null
+++ b/dev-tools/aws-jmh/group_vars/all.yml
@@ -0,0 +1,65 @@
+# cloudformation stack name
+stack_name: lucene-jmh
+
+# region to launch instances, make sure they are supported!
+region: us-east-1
+
+# ssh keypair to use for accessing instances
+ssh_key: ~/.ssh/id_ed25519
+
+# user to create remotely
+user: lucene
+
+# ami to use
+x86_ami: ami-04cb4ca688797756f
+arm_ami: ami-06f9c0b2ce386dda7
+
+# instance types to spin up
+instances:
+ - name: haswell
+ type: c4.large
+ ami: "{{ x86_ami }}"
+ - name: cascadelake
+ type: c5.large
+ ami: "{{ x86_ami }}"
+ - name: icelake
+ type: c6i.large
+ ami: "{{ x86_ami }}"
+ - name: sapphirerapids
+ type: c7i.large
+ ami: "{{ x86_ami }}"
+ - name: zen2
+ type: c5a.large
+ ami: "{{ x86_ami }}"
+ - name: zen3
+ type: c6a.large
+ ami: "{{ x86_ami }}"
+ - name: zen4
+ type: c7a.large
+ ami: "{{ x86_ami }}"
+ - name: graviton2
+ type: c6g.large
+ ami: "{{ arm_ami }}"
+ - name: graviton3
+ type: c7g.large
+ ami: "{{ arm_ami }}"
+
+jdk_version: 21.0.1
+x86_jdk_url: "https://download.java.net/java/GA/jdk{{ jdk_version }}/415e3f918a1f4062a0074a2794853d0d/12/GPL/openjdk-{{ jdk_version }}_linux-x64_bin.tar.gz"
+arm_jdk_url: "https://download.java.net/java/GA/jdk{{ jdk_version }}/415e3f918a1f4062a0074a2794853d0d/12/GPL/openjdk-{{ jdk_version }}_linux-aarch64_bin.tar.gz"
+jdk_url: "{{ arm_jdk_url if ansible_architecture == 'aarch64' else x86_jdk_url }}"
+
+main_branch: "apache:main"
+patch_branch: "{{ lookup('env', 'PATCH_BRANCH') }}"
+
+assemble_command: ./gradlew -p lucene/benchmark-jmh assemble
+jmh_command: java -jar lucene/benchmark-jmh/build/benchmarks/lucene-benchmark-jmh-10.0.0-SNAPSHOT.jar -rf text
+jmh_args: "{{ lookup('env', 'JMH_ARGS', default='-p size=1024') }}"
+
+# services that cause noise and don't help us
+disable_services:
+ - atd.service
+ - logrotate.timer
+ - sysstat-collect.timer
+ - sysstat-summary.timer
+ - systemd-tmpfiles-clean.timer
diff --git a/dev-tools/aws-jmh/playbook.yml b/dev-tools/aws-jmh/playbook.yml
new file mode 100644
index 00000000000..197a415c23e
--- /dev/null
+++ b/dev-tools/aws-jmh/playbook.yml
@@ -0,0 +1,155 @@
+
+# spin up aws hardware
+- hosts: localhost
+ gather_facts: false
+ tasks:
+ - name: Lookup default VPC
+ ec2_vpc_net_info:
+ region: "{{ region }}"
+ filters:
+ isDefault: true
+ register: vpc_info
+
+ - name: Create cloudformation stack
+ cloudformation:
+ stack_name: "{{ stack_name }}"
+ region: "{{ region }}"
+ template_body: "{{ lookup('template', 'cloudformation.yml.j2') }}"
+
+ - name: Gather instance details
+ ec2_instance_info:
+ region: "{{ region }}"
+ filters:
+ instance-state-name: ["pending", "running"]
+ "tag:aws:cloudformation:stack-name": "{{ stack_name }}"
+ register: ec2_instances
+
+ - name: Add instances to inventory
+ add_host:
+ name: "{{ item['tags']['ShortName'] }}"
+ ansible_host: "{{ item['public_ip_address'] }}"
+ ansible_user: "{{ user }}"
+ ansible_ssh_private_key_file: "{{ ssh_key }}"
+ args: "{{ item }}"
+ with_items: "{{ ec2_instances.instances }}"
+
+- hosts: all
+ gather_facts: false
+ tasks:
+ - name: Wait for connection
+ wait_for_connection:
+
+ - name: Gather facts
+ setup:
+
+ - name: Configure kernel
+ command:
+ argv:
+ - grubby
+ - --remove-args=selinux_1 security=selinux quiet
+ - --args=mitigations=0 random.trust_cpu=1 loglevel=7 selinux=0
+ - --update-kernel=ALL
+ become: true
+
+ - name: Install packages
+ package:
+ name:
+ - git
+ become: true
+
+ - name: Disable noisy services
+ command:
+ cmd: "systemctl disable --now {{ item }}"
+ loop: "{{ disable_services }}"
+ failed_when: false
+ become: true
+
+ - name: Reboot machine
+ reboot:
+ become: true
+
+ - name: Download JDK
+ unarchive:
+ src: "{{ jdk_url }}"
+ dest: .
+ remote_src: true
+
+ - name: Configure JDK
+ blockinfile:
+ path: .bashrc
+ block: |
+ export JAVA_HOME=/home/{{ user }}/jdk-{{ jdk_version }}
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ - name: Configure Gradle
+ blockinfile:
+ path: .gradle/gradle.properties
+ create: true
+ block: |
+ org.gradle.daemon=false
+
+ - name: Checkout main
+ git:
+ repo: "git@github.com:{{ (main_branch | split(':'))[0] }}/lucene.git"
+ version: "{{ (main_branch | split(':'))[1] }}"
+ dest: main
+ accept_newhostkey: true
+ depth: 1
+
+ - name: Checkout patch
+ git:
+ repo: "git@github.com:{{ (patch_branch | split(':'))[0] }}/lucene.git"
+ version: "{{ (patch_branch | split(':'))[1] }}"
+ dest: patch
+ accept_newhostkey: true
+ depth: 1
+
+ - name: Assemble Sources
+ command:
+ cmd: "{{ assemble_command }}"
+ chdir: "{{ item }}"
+ loop:
+ - main
+ - patch
+
+ - name: Run benchmark
+ command:
+ cmd: "{{ jmh_command }} -rff ~/{{ item }}.txt {{ jmh_args }}"
+ chdir: "{{ item }}"
+ loop:
+ - main
+ - patch
+
+ - name: Read main results
+ command:
+ cmd: "cat main.txt"
+ register: main_out
+
+ - name: Read patch results
+ command:
+ cmd: "cat patch.txt"
+ register: patch_out
+
+ - name: Write Report
+ copy:
+ dest: "build/{{ inventory_hostname }}.log"
+ content: |
+
+ {{ inventory_hostname }}: `{{ ansible_processor }}`
+
+ main
+ ```
+ {{ main_out.stdout }}
+ ```
+ patch
+ ```
+ {{ patch_out.stdout }}
+ ```
+ delegate_to: localhost
+
+- hosts: localhost
+ gather_facts: false
+ tasks:
+ - name: Create combined report
+ shell:
+ cmd: "cat build/*.log > build/report.txt"
diff --git a/dev-tools/aws-jmh/requirements.txt b/dev-tools/aws-jmh/requirements.txt
new file mode 100644
index 00000000000..09175f717fa
--- /dev/null
+++ b/dev-tools/aws-jmh/requirements.txt
@@ -0,0 +1,4 @@
+# automate botnet with ansible
+ansible<9
+# aws api
+boto3<2
diff --git a/gradle/validation/rat-sources.gradle b/gradle/validation/rat-sources.gradle
index 4d9759188cb..3c215c35223 100644
--- a/gradle/validation/rat-sources.gradle
+++ b/gradle/validation/rat-sources.gradle
@@ -82,6 +82,9 @@ allprojects {
// projects. Include their sources in the scan.
include "buildSrc/src/**"
include "dev-tools/missing-doclet/src/**"
+
+ // do not let RAT attempt to scan a python venv, it gets lost and confused...
+ exclude "dev-tools/aws-jmh/build/**"
break
case ":lucene:analysis:morfologik":