You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by md...@apache.org on 2018/04/19 12:01:22 UTC
[incubator-openwhisk] branch master updated: Fix vagrant not use
runc (#3546)
This is an automated email from the ASF dual-hosted git repository.
mdeuser pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new e9aa8e9 Fix vagrant not use runc (#3546)
e9aa8e9 is described below
commit e9aa8e908da06719e501bffa2be6d82ff6f4e563
Author: Carlos Santana <cs...@apache.org>
AuthorDate: Thu Apr 19 08:01:18 2018 -0400
Fix vagrant not use runc (#3546)
* fix vagrant to not use runc
* setup gitattributes for auth files
* address review comments for readme
---
.gitattributes | 4 +
.gitignore | 1 +
ansible/environments/vagrant/group_vars/all | 42 ++++++++++
ansible/environments/vagrant/hosts | 29 +++++++
ansible/files/auth.guest | 2 +-
ansible/files/auth.whisk.system | 2 +-
tools/ubuntu-setup/docker-xenial.sh | 4 -
tools/ubuntu-setup/docker.sh | 3 -
tools/vagrant/README.md | 119 +++++++++++++---------------
tools/vagrant/Vagrantfile | 48 +++++++----
10 files changed, 164 insertions(+), 90 deletions(-)
diff --git a/.gitattributes b/.gitattributes
index b19d1c0..71665f0 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -29,3 +29,7 @@ gradlew text eol=lf
core/javaAction/proxy/gradlew text eol=lf
tools/vagrant/hello text eol=lf
sdk/docker/client/action text eol=lf
+
+# auth files with default api keys
+ansible/files/auth.guest text eol=lf
+ansible/files/auth.whisk.system text eol=lf
diff --git a/.gitignore b/.gitignore
index aab3af0..5ba08fc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,6 +17,7 @@ results
!/ansible/environments/docker-machine
!/ansible/environments/local
!/ansible/environments/mac
+!/ansible/environments/vagrant
# Eclipse
bin/
diff --git a/ansible/environments/vagrant/group_vars/all b/ansible/environments/vagrant/group_vars/all
new file mode 100755
index 0000000..2015f0f
--- /dev/null
+++ b/ansible/environments/vagrant/group_vars/all
@@ -0,0 +1,42 @@
+whisk_version_name: local
+openwhisk_tmp_dir: "{{ lookup('env', 'OPENWHISK_TMP_DIR')|default('/tmp', true) }}"
+config_root_dir: "{{ openwhisk_tmp_dir }}/wskconf"
+whisk_logs_dir: "{{ openwhisk_tmp_dir }}/wsklogs"
+docker_registry: ""
+docker_dns: ""
+runtimes_bypass_pull_for_local_images: true
+invoker_use_runc: "{{ ansible_distribution != 'MacOSX' }}"
+
+db_prefix: whisk_local_
+
+# Auto lookup to find the db credentials
+db_provider: "{{ lookup('ini', 'db_provider section=db_creds file={{ playbook_dir }}/db_local.ini') }}"
+db_username: "{{ lookup('ini', 'db_username section=db_creds file={{ playbook_dir }}/db_local.ini') }}"
+db_password: "{{ lookup('ini', 'db_password section=db_creds file={{ playbook_dir }}/db_local.ini') }}"
+db_protocol: "{{ lookup('ini', 'db_protocol section=db_creds file={{ playbook_dir }}/db_local.ini') }}"
+db_host: "{{ lookup('ini', 'db_host section=db_creds file={{ playbook_dir }}/db_local.ini') }}"
+db_port: "{{ lookup('ini', 'db_port section=db_creds file={{ playbook_dir }}/db_local.ini') }}"
+
+# API GW connection configuration
+apigw_auth_user: ""
+apigw_auth_pwd: ""
+apigw_host_v2: "http://{{ groups['apigateway']|first }}:{{apigateway.port.api}}/v2"
+
+invoker_allow_multiple_instances: true
+
+# Set kafka configuration
+kafka_heap: '512m'
+kafka_topics_completed_retentionBytes: 104857600
+kafka_topics_completed_retentionMS: 300000
+kafka_topics_health_retentionBytes: 104857600
+kafka_topics_health_retentionMS: 300000
+kafka_topics_invoker_retentionBytes: 104857600
+kafka_topics_invoker_retentionMS: 300000
+
+env_hosts_dir: "{{ playbook_dir }}/environments/local"
+
+controller_protocol: "http"
+
+cli_installation_mode: "local"
+
+invoker_use_runc: false
diff --git a/ansible/environments/vagrant/hosts b/ansible/environments/vagrant/hosts
new file mode 100644
index 0000000..b630812
--- /dev/null
+++ b/ansible/environments/vagrant/hosts
@@ -0,0 +1,29 @@
+; the first parameter in a host is the inventory_hostname
+
+; used for local actions only
+ansible ansible_connection=local
+
+[edge]
+172.17.0.1 ansible_host=172.17.0.1 ansible_connection=local
+
+[controllers]
+controller0 ansible_host=172.17.0.1 ansible_connection=local
+;
+[kafkas]
+kafka0 ansible_host=172.17.0.1 ansible_connection=local
+
+[zookeepers:children]
+kafkas
+
+[invokers]
+invoker0 ansible_host=172.17.0.1 ansible_connection=local
+
+; db group is only used if db_provider is CouchDB
+[db]
+172.17.0.1 ansible_host=172.17.0.1 ansible_connection=local
+
+[redis]
+172.17.0.1 ansible_host=172.17.0.1 ansible_connection=local
+
+[apigateway]
+172.17.0.1 ansible_host=172.17.0.1 ansible_connection=local
diff --git a/ansible/files/auth.guest b/ansible/files/auth.guest
index 3156274..39eda06 100644
--- a/ansible/files/auth.guest
+++ b/ansible/files/auth.guest
@@ -1 +1 @@
-23bc46b1-71f6-4ed5-8c54-816aa4f8c502:123zO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP
+23bc46b1-71f6-4ed5-8c54-816aa4f8c502:123zO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP
\ No newline at end of file
diff --git a/ansible/files/auth.whisk.system b/ansible/files/auth.whisk.system
index e44545b..dbcd279 100644
--- a/ansible/files/auth.whisk.system
+++ b/ansible/files/auth.whisk.system
@@ -1 +1 @@
-789c46b1-71f6-4ed5-8c54-816aa4f8c502:abczO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP
+789c46b1-71f6-4ed5-8c54-816aa4f8c502:abczO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP
\ No newline at end of file
diff --git a/tools/ubuntu-setup/docker-xenial.sh b/tools/ubuntu-setup/docker-xenial.sh
index 812a0c2..fa3a314 100644
--- a/tools/ubuntu-setup/docker-xenial.sh
+++ b/tools/ubuntu-setup/docker-xenial.sh
@@ -51,10 +51,6 @@ sudo apt-get install -y docker-ce # Replace with lines above to lock in version
sudo -E bash -c 'echo '\''DOCKER_OPTS="-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock --storage-driver=aufs"'\'' >> /etc/default/docker'
sudo gpasswd -a "$(whoami)" docker
-# Set DOCKER_HOST as an environment variable
-sudo -E bash -c 'echo '\''export DOCKER_HOST="tcp://0.0.0.0:4243"'\'' >> /etc/bash.bashrc'
-source /etc/bash.bashrc
-
sudo service docker restart
# do not run this command without a vagrant reload during provisioning
diff --git a/tools/ubuntu-setup/docker.sh b/tools/ubuntu-setup/docker.sh
index de3a76c..caa74f6 100755
--- a/tools/ubuntu-setup/docker.sh
+++ b/tools/ubuntu-setup/docker.sh
@@ -39,9 +39,6 @@ sudo apt-mark hold docker-engine
sudo -E bash -c 'echo '\''DOCKER_OPTS="-H tcp://0.0.0.0:4243 -H unix:///var/run/docker.sock --storage-driver=aufs"'\'' >> /etc/default/docker'
sudo gpasswd -a "$(whoami)" docker
-# Set DOCKER_HOST as an environment variable
-sudo -E bash -c 'echo '\''export DOCKER_HOST="tcp://0.0.0.0:4243"'\'' >> /etc/bash.bashrc'
-source /etc/bash.bashrc
sudo service docker restart
diff --git a/tools/vagrant/README.md b/tools/vagrant/README.md
index 3e25b19..18a216b 100644
--- a/tools/vagrant/README.md
+++ b/tools/vagrant/README.md
@@ -1,16 +1,16 @@
<!--
#
-# Licensed to the Apache Software Foundation (ASF) under one or more contributor
-# license agreements. See the NOTICE file distributed with this work for additional
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with this work for additional
# information regarding copyright ownership. The ASF licenses this file to you
-# under the Apache License, Version 2.0 (the # "License"); you may not use this
-# file except in compliance with the License. You may obtain a copy of the License
+# under the Apache License, Version 2.0 (the # "License"); you may not use this
+# file except in compliance with the License. You may obtain a copy of the License
# at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#
@@ -21,8 +21,8 @@
The following instructions were tested on Mac OS X El Capitan, Ubuntu 16.04 LTS.
## Requirements
-- Install [VirtualBox](https://www.virtualbox.org/wiki/Downloads) (tested with version 5.2.6)
-- Install [Vagrant](https://www.vagrantup.com/downloads.html) (tested with version 2.02)
+- Install [VirtualBox](https://www.virtualbox.org/wiki/Downloads) (tested with version 5.2.8)
+- Install [Vagrant](https://www.vagrantup.com/downloads.html) (tested with version 2.0.3)
## Setup
@@ -88,7 +88,7 @@ follow the manual process to build and deploy in
**Tip:**
By default, each `docker` command will timeout after 840 seconds (14 minutes).
If you're on a really slow connection, this might be too short. You can modify
-the timeout value in [docker.gradle](../../../gradle/docker.gradle#L22) as
+the timeout value in [docker.gradle](../../gradle/docker.gradle#L22).
### Using CLI from outside the VM
@@ -98,8 +98,10 @@ machine. The IP address of the virtual machine accessible from outside is
address will conflict, use `vagrant suspend` before starting another VM with the
same IP address.
-The CLI is available in `../../bin`. There you will find binaries specific to
-various operating systems and architectures (e.g. `../../bin/mac/amd64/wsk`).
+The CLI is available in `../../bin`.
+The CLI `../../bin/wsk` is for Linux amd64.
+The CLI for other operating systems and architectures can be found under `../../bin/openwhisk-cli/build/`
+
When using the CLI with a local deployment of OpenWhisk (which provides an
insecure/self-signed SSL certificate), you must use the argument `-i` to permit
an insecure HTTPS connection to OpenWhisk. This should be used for development
@@ -137,7 +139,7 @@ sdk` will not work, so you need to pass use `wsk -i --apihost 192.168.33.16 sdk
**Note:**
-To connect to a different host API (i.e. bluemix.net) with the CLI, you will
+To connect to a different host API (i.e. openwhisk.example.com) with the CLI, you will
need to configure the CLI with new values for _apihost_, and _auth_ key.
### Use the wsk CLI inside the VM
@@ -156,6 +158,14 @@ vagrant ssh
wsk action invoke /whisk.system/utils/echo -p message hello --result
```
+## Other Runntimes
+The default vagrant deploy only deploys nodejs:6 runtime kind, because the image runs out of space if all runtimes are built.
+To add a runtime, you need to build the runtime image for example
+```
+wskdev python3action
+```
+To get a list of other available runtimes use `wskdev -c`
+
## Running OpenWhisk tests
```
vagrant ssh
@@ -176,46 +186,49 @@ cd ${OPENWHISK_HOME}
./gradlew distDocker
```
-## Safe Re-deploy (after VM restart)
-
-If you restart the VM (e.g., `vagrant reload`), it may be necessary to refresh
-the OpenWhisk deployment. You can do this in a way that does not reload the data
-store container.
+## Using docker-runc
+Only for experimental use:
+To use docker-runc in the invoker, the version of docker-runc needs to match the version of docker engine on the host.
+Get the version of the docker engine on the host like the following:
+```
+$ docker version | grep Version
+Version: 18.03.0-ce
+```
+You need to use the same version for docker-runc in the Invoker, to use a newer version of docker-runc in the invoker, update the invoker Dockerfile.
+1. compare the docker-runc version obtained on the local system against the docker-runc configured for the invoker
+2. if the versions are different, only then do you need to update the invoker dockerfile to point to the matching docker download
+Edit the [core/invoker/Dockefile](../../core/invoker/Dockefile)
+Update the variable with the version
```
-vagrant ssh
-cd ${ANSIBLE_HOME}
-# teardown all containers expect couchdb container
-ansible-playbook -i environments/local openwhisk.yml -e mode=clean
-# deploy openwhisk containers
-ansible-playbook -i environments/local openwhisk.yml
+ENV DOCKER_VERSION 18.03.0-ce
+```
+Then update line with the curl download command like
+```
+RUN curl -sSL -o docker-${DOCKER_VERSION}.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz && \
```
+Notice that the hostname where to download the cli is different for newer versions.
-The following commands are helpful to deploy a fresh OpenWhisk and data store
-after booting a new VM using `vagrant up`.
+Then update the ansible configuration to enable the use of runc, edit [](../../ansible/environments/vagrant/group_vars/all)
+```
+invoker_use_runc: true
+```
-### Teardown and Deploy (refresh the data store)
-Use ansible to re-deploy OpenWhisk from inside the VM
-To deploy a new code base you need to [re-build OpenWhisk](#build-openwhisk)
-first
+Then rebuild and redeploy the invoker component
+```
+wskdev invoker
+```
+### Teardown and Deploy
+The following commands are helpful to deploy a fresh OpenWhisk and data store
```
vagrant ssh
cd ${ANSIBLE_HOME}
-# teardown all deployed containers
-ansible-playbook -i environments/local teardown.yml
-# deploy couchdb container
-ansible-playbook -i environments/local couchdb.yml
-# initialize db with guest/system keys
-ansible-playbook -i environments/local initdb.yml
-# recreate main db for entities
-ansible-playbook -i environments/local wipe.yml
+# teardown all containers
+wskdev teardown
# deploy openwhisk containers
-ansible-playbook -i environments/local openwhisk.yml
-# install catalog
-ansible-playbook -i environments/local postdeploy.yml
+wskdev fresh
```
-
**Tip**
Do not restart the VM using Virtual Box tools, and always use `vagrant` from the
command line: `vagrant up` to start the VM and `vagrant reload` to restart it.
@@ -245,29 +258,7 @@ create a new key.
vagrant ssh
wskadmin user create <subject>
```
-
-This command will create a new _subject_ with the authorization key shown on the
-console once you run `wskadmin`. This key is required when making API calls to
-OpenWhisk, or when using the command line interface (CLI). The namespace is the
-same as the `<subject>` name used to create the key.
-
-A namespace allows two or more subjects to share resources. Each subject will
-have their own authorization key to work with resources in a namespace, but will
-have equal rights to the namespace.
-
-```
-vagrant ssh
-wskadmin user create <subject> -ns <namespace>
-```
-
-The same tool may be used to remove a subject from a namespace or to delete a
-subject entirely.
-
-```
-vagrant ssh
-wskadmin user delete <subject> -ns <namespace> # removes <subject> from <namespace>
-wskadmin user delete <subject> # deletes <subject>
-```
+For more information on `wskadmin` check the [documentation](../admin).
## SSL certificate configuration (Optional)
diff --git a/tools/vagrant/Vagrantfile b/tools/vagrant/Vagrantfile
index d1da5af..2689ec8 100644
--- a/tools/vagrant/Vagrantfile
+++ b/tools/vagrant/Vagrantfile
@@ -56,6 +56,7 @@ Vagrant.configure('2') do |config|
HOME=/home/vagrant
OPENWHISK_HOME=${HOME}/openwhisk
+ OPENWHISK_CLI=${OPENWHISK_HOME}/bin/openwhisk-cli
OPENWHISK_TMP_DIR=${HOME}/.openwhisk/ansible/tmp
ANSIBLE_HOME=${OPENWHISK_HOME}/ansible
ANSIBLE_REMOTE_TEMP=${ANSIBLE_HOME}/tmp
@@ -64,10 +65,10 @@ Vagrant.configure('2') do |config|
echo ANSIBLE_HOME=${ANSIBLE_HOME} >> /etc/environment
echo ANSIBLE_REMOTE_TEMP=${ANSIBLE_REMOTE_TEMP} >> /etc/environment
echo OPENWHISK_HOME=${OPENWHISK_HOME} >> /etc/environment
+ echo OPENWHISK_CLI=${OPENWHISK_CLI} >> /etc/environment
echo OPENWHISK_TMP_DIR=${OPENWHISK_TMP_DIR} >> /etc/environment
echo PATH=${PATH}:${HOME}/bin:${OPENWHISK_HOME}/tools/build >> /etc/environment
-
- cd /home/vagrant/openwhisk
+ cd ${OPENWHISK_HOME}
cd tools/ubuntu-setup
su vagrant -c 'source all.sh'
echo "`date`: ubuntu-setup-end" >> /tmp/vagrant-times.txt
@@ -92,8 +93,8 @@ Vagrant.configure('2') do |config|
fi
cd /home/vagrant/openwhisk/ansible
- su vagrant -c 'ansible-playbook -i environments/local setup.yml'
- su vagrant -c 'ansible-playbook -i environments/local prereq.yml'
+ su vagrant -c 'ansible-playbook -i environments/vagrant setup.yml'
+ su vagrant -c 'ansible-playbook -i environments/vagrant prereq.yml'
SCRIPT
end
@@ -101,9 +102,9 @@ Vagrant.configure('2') do |config|
echo "`date`: couchdb-setup-start" >> /tmp/vagrant-times.txt
export ANSIBLE_REMOTE_TEMP=/home/vagrant/openwhisk/ansible/tmp
cd /home/vagrant/openwhisk/ansible
- su vagrant -c 'ansible-playbook -i environments/local setup.yml'
- su vagrant -c 'ansible-playbook -i environments/local prereq.yml'
- su vagrant -c 'ansible-playbook -i environments/local couchdb.yml'
+ su vagrant -c 'ansible-playbook -i environments/vagrant setup.yml'
+ su vagrant -c 'ansible-playbook -i environments/vagrant prereq.yml'
+ su vagrant -c 'ansible-playbook -i environments/vagrant couchdb.yml'
echo "`date`: couchdb-setup-end" >> /tmp/vagrant-times.txt
COUCHDB
end
@@ -111,38 +112,49 @@ Vagrant.configure('2') do |config|
config.vm.provision "initdb", type: "shell", keep_color: true, inline: <<-INITDB
export ANSIBLE_REMOTE_TEMP=/home/vagrant/openwhisk/ansible/tmp
cd /home/vagrant/openwhisk/ansible
- su vagrant -c 'ansible-playbook -i environments/local initdb.yml'
+ su vagrant -c 'ansible-playbook -i environments/vagrant initdb.yml'
INITDB
# Build and Deploy
config.vm.provision "script_end", type: "shell", keep_color: true, inline: <<-SCRIPT, env: {"WHISK_IP" => MACHINE_IP}
set -e
set -x
- echo "`date`: build-start" >> /tmp/vagrant-times.txt
HOME=/home/vagrant
source /etc/environment
+
# Build OpenWhisk using gradle
+ echo "`date`: build-core-start" >> /tmp/vagrant-times.txt
cd ${OPENWHISK_HOME}
- su vagrant -c './gradlew distDocker'
+ su vagrant -c './gradlew :core:controller:distDocker :core:invoker:distDocker :actionRuntimes:nodejs6Action:distDocker'
+ echo "`date`: build-core-end" >> /tmp/vagrant-times.txt
+
+ # Clone and Build CLI
+ echo "`date`: build-cli-start" >> /tmp/vagrant-times.txt
+ cd ${OPENWHISK_HOME}/bin
+ rm -rf openwhisk-cli
+ su vagrant -c 'git clone --depth=1 https://github.com/apache/incubator-openwhisk-cli.git openwhisk-cli'
+ cd openwhisk-cli
+ su vagrant -c './gradlew releaseBinaries'
+ echo "`date`: build-cli-end" >> /tmp/vagrant-times.txt
- echo "`date`: deploy-start" >> /tmp/vagrant-times.txt
# Deploy OpenWhisk using ansible
+ echo "`date`: deploy-start" >> /tmp/vagrant-times.txt
cd ${ANSIBLE_HOME}
- su vagrant -c 'ansible-playbook -i environments/local wipe.yml'
- su vagrant -c 'ansible-playbook -i environments/local apigateway.yml'
- su vagrant -c 'ansible-playbook -i environments/local openwhisk.yml'
- su vagrant -c 'ansible-playbook -i environments/local postdeploy.yml'
+ su vagrant -c 'ansible-playbook -i environments/vagrant wipe.yml'
+ su vagrant -c 'ansible-playbook -i environments/vagrant apigateway.yml'
+ su vagrant -c 'ansible-playbook -i environments/vagrant openwhisk.yml -e invoker_use_runc=False'
+ su vagrant -c 'ansible-playbook -i environments/vagrant postdeploy.yml'
# Setup OpenWhisk CLI
su vagrant -c 'mkdir ${HOME}/bin'
su vagrant -c 'ln -s ${OPENWHISK_HOME}/tools/admin/wskadmin ${HOME}/bin/wskadmin'
- su vagrant -c 'ln -s ${OPENWHISK_HOME}/tools/build/redo ${HOME}/bin/wskdev'
+ echo ${OPENWHISK_HOME}/tools/build/redo -t vagrant '"$@"' > ${HOME}/bin/wskdev
+ chmod +x ${HOME}/bin/wskdev
echo ${OPENWHISK_HOME}/bin/wsk -i '"$@"' > ${HOME}/bin/wsk
chmod +x ${HOME}/bin/wsk
chown vagrant:vagrant ${HOME}/bin/wsk
PATH=${PATH}:${HOME}/bin
echo 'eval "`wsk sdk install bashauto --stdout`"'>>${HOME}/.bashrc
-
# Run OpenWhisk CLI
touch $HOME/.wskprops
chown -R vagrant:vagrant $HOME
@@ -156,9 +168,11 @@ Vagrant.configure('2') do |config|
if ENV['gui'] == 'true'
# Install ubuntu-desktop and virtualbox additions
config.vm.provision "shell", keep_color: true, inline: "apt-get update"
+ config.vm.provision "shell", keep_color: true, inline: "apt-get install -y xfce4"
config.vm.provision "shell", keep_color: true, inline: "apt-get install -y virtualbox-guest-dkms"
config.vm.provision "shell", keep_color: true, inline: "apt-get install -y virtualbox-guest-utils"
config.vm.provision "shell", keep_color: true, inline: "apt-get install -y virtualbox-guest-x11"
+ config.vm.provision "shell", keep_color: true, inline: "touch /etc/X11/Xwrapper.config"
config.vm.provision "shell", keep_color: true, inline: "sed -i 's/allowed_users=.*$/allowed_users=anybody/' /etc/X11/Xwrapper.config"
#the ; true is used to ignore errors from ubuntu-desktop, vagrant reload should be done after provisioning
config.vm.provision "shell", keep_color: true, inline: "sudo sh -c 'apt-get install -y ubuntu-desktop ; true'"
--
To stop receiving notification emails like this one, please contact
mdeuser@apache.org.