You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by "Yingyi Bu (Code Review)" <do...@asterixdb.incubator.apache.org> on 2017/01/30 23:33:48 UTC
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Yingyi Bu has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1475
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Add AWS automation scripts to asterix-server.
- Allows users to customize an AWS-based instance by editing
conf/aws_settings.yml and conf/instance_settings.yml.
- Starts an AWS cluster, install JDK and AsterixDB automatically.
Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
---
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
M asterixdb/asterix-server/pom.xml
M asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
A asterixdb/asterix-server/src/main/aws/README
A asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
A asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
A asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
A asterixdb/asterix-server/src/main/aws/bin/start.sh
A asterixdb/asterix-server/src/main/aws/bin/stop.sh
A asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
A asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
13 files changed, 550 insertions(+), 5 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/75/1475/1
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index faacc64..68ad80c 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -18,10 +18,10 @@
*/
package org.apache.asterix.common.config;
+import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
+
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.util.StorageUtil;
-
-import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
public class StorageProperties extends AbstractProperties {
@@ -40,7 +40,6 @@
private static final String STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY =
"storage.metadata.memorycomponent.numpages";
- private static final int STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 256; // ... so 32MB components
private static final String STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_KEY = "storage.memorycomponent.numcomponents";
private static final int STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_DEFAULT = 2; // 2 components
@@ -53,6 +52,7 @@
private final long storageBufferCacheSizeDefault;
private final int storageMemoryComponentNumPages;
+ private final int storageMetadataMemoryComponentNumPages;
private final long storageMemorycomponentGlobalbudgetDefault;
public StorageProperties(PropertiesAccessor accessor) {
@@ -68,6 +68,10 @@
// for a dataset, including data and indexes.
storageMemoryComponentNumPages = (int) (storageMemorycomponentGlobalbudgetDefault
/ (16 * getMemoryComponentPageSize()));
+ // By default, uses the min of 1/64 of the storageMemorycomponentGlobalbudgetDefault and 256 pages
+ // for the write buffer budget for a metadata dataset, including data and indexes.
+ storageMetadataMemoryComponentNumPages = Math
+ .min((int) (storageMemorycomponentGlobalbudgetDefault / (64 * getMemoryComponentPageSize())), 256);
}
@PropertyKey(STORAGE_BUFFERCACHE_PAGESIZE_KEY)
@@ -107,8 +111,7 @@
@PropertyKey(STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY)
public int getMetadataMemoryComponentNumPages() {
return accessor.getProperty(STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY,
- STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_DEFAULT,
- PropertyInterpreters.getIntegerPropertyInterpreter());
+ storageMetadataMemoryComponentNumPages, PropertyInterpreters.getIntegerPropertyInterpreter());
}
@PropertyKey(STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_KEY)
diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml
index 3454428..f91ed64 100644
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@ -493,5 +493,9 @@
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
index b968ae1..32a0180 100644
--- a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
+++ b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
@@ -52,6 +52,23 @@
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
+ <directory>src/main/aws</directory>
+ <outputDirectory>aws</outputDirectory>
+ <excludes>
+ <exclude>**/*.sh</exclude>
+ </excludes>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/aws</directory>
+ <outputDirectory>aws</outputDirectory>
+ <includes>
+ <include>**/*.sh</include>
+ </includes>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
<directory>target/appassembler/repo</directory>
<outputDirectory>repo</outputDirectory>
</fileSet>
diff --git a/asterixdb/asterix-server/src/main/aws/README b/asterixdb/asterix-server/src/main/aws/README
new file mode 100644
index 0000000..73204fe
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/README
@@ -0,0 +1,46 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+To start an AWS cluster, you need to do the following steps:
+
+1. Create an AWS account and an IAM user.
+
+2. Retrieve your AWS key pair name and fill that after "keypair:" in conf/aws_settings.yml;
+ retrieve your AWS "access key ID" and fill that after "access_key_id:" in conf/aws_settings.yml;
+ retrieve your AWS "secret access key" and fill that after "secret_access_key:" in conf/aws_settings.yml.
+ Note that you can only read or download "access key ID" and "secret access key" once from your AWS console.
+ If you forget them, you have to create new keys again and delete the old ones.
+
+3. Customize other settings in conf/aws_settings.yml and conf/instance_settings.yml to whatever you want.
+
+4. Configure your ssh setting by editing ~/.ssh/config and adding the following entry:
+ Host *.amazonaws.com
+ IdentityFile <path_of_private_key>
+ Note that <path_of_private_key> should be replaced by the path to the file that stores the private key for the key
+ pair that you uploaded to AWS and used in step 2. For example:
+ Host *.amazonaws.com
+ IdentityFile ~/.ssh/id_rsa
+
+5. Launch your cluster instance on AWS:
+ bin/start.sh
+ Now you can use the AWS-based instance.
+
+6. Terminate the cluster instance on AWS:
+ bin/stop.sh
+ Note that it will destroy the instance and terminate all ec2 instances that you launched in step 5.
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml b/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
new file mode 100644
index 0000000..e30731c
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
@@ -0,0 +1,47 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Start AWS cluster
+ hosts: localhost
+ gather_facts: false
+ tasks:
+ - include_vars: ../conf/aws_settings.yml
+ - name: Launch all instances
+ ec2:
+ key_name: "{{ keypair }}"
+ group: "{{ group }}"
+ instance_type: "{{ instance_type }}"
+ image: "{{ image }}"
+ wait: true
+ region: "{{ region }}"
+ aws_access_key: "{{ access_key_id }}"
+ aws_secret_key: "{{ secret_access_key }}"
+ exact_count: "{{ count }}"
+ count_tag:
+ Name: "{{ tag }}"
+ instance_tags:
+ Name: "{{ tag }}"
+ register: ec2
+ - file:
+ path: /tmp/asterixdb
+ state: directory
+ mode: 0755
+ - name: Output the information of all nodes for future processing
+ local_action: copy content="{{ ec2|to_json }}" dest=/tmp/asterixdb/nodes
+
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml b/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
new file mode 100644
index 0000000..e1bdc86
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
@@ -0,0 +1,35 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Stop AWS cluster
+ hosts: localhost
+ gather_facts: false
+ tasks:
+ - include_vars: ../conf/aws_settings.yml
+ - name: Stop all instance
+ ec2:
+ key_name: "{{ keypair }}"
+ group: "{{ group }}"
+ instance_type: "{{ instance_type }}"
+ image: "{{ image }}"
+ wait: true
+ region: "{{ region }}"
+ aws_access_key: "{{ access_key_id }}"
+ aws_secret_key: "{{ secret_access_key }}"
+ exact_count: 0
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml b/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
new file mode 100644
index 0000000..1718ade
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
@@ -0,0 +1,68 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- hosts: ncs
+ tasks:
+ - include_vars: ../conf/instance_settings.yml
+ - name: Download JDK
+ shell: "wget -q --tries=5 --no-cookies --no-check-certificate --header \
+ \"Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie\" \
+ \"http://download.oracle.com/otn-pub/java/jdk/8u60-b27/jre-8u60-linux-x64.rpm\""
+
+ - name: Install JDK
+ shell: sudo yum -y localinstall jre-8u60-linux-x64.rpm
+
+ - name: Ensure the working directory exists
+ file:
+ path: "{{ binarydir }}"
+ state: directory
+
+ - name: Unzip binary
+ unarchive:
+ src: "{{ lookup('pipe', 'ls -1 ' + srcpattern) }}"
+ dest: "{{ binarydir }}"
+
+ - name: Ensure the log directory exists
+ file:
+ path: "{{ binarydir }}/logs"
+ state: directory
+
+ - name: Ensure the io device directory exit
+ file:
+ path: "{{ basedir }}/iodevice"
+ state: directory
+
+ - name: Start NC Service
+ shell: nohup "{{ ncservice }}" &> "{{ binarydir }}/logs/ncservice.log" &
+ async: 10
+ poll: 0
+
+- hosts: cc
+ tasks:
+ - include_vars: ../conf/instance_settings.yml
+ - name: Copy cluster config to CC
+ copy:
+ src: /tmp/asterixdb/cc.conf
+ dest: "{{ basedir }}/cc.conf"
+
+ - name: Start CC
+ shell: nohup "{{ cc }}" -config-file "{{ basedir }}/cc.conf" &> "{{ binarydir }}/logs/cc.log" &
+ async: 10
+ poll: 0
+
diff --git a/asterixdb/asterix-server/src/main/aws/bin/start.sh b/asterixdb/asterix-server/src/main/aws/bin/start.sh
new file mode 100755
index 0000000..be33428
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/bin/start.sh
@@ -0,0 +1,38 @@
+#!/bin/bash -x
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+
+# starts an AWS cluster
+ansible-playbook -i "localhost," ansible/aws_start.yml
+
+# generates an Ansible inventory file and an AsterixDB configuration file
+temp=/tmp/asterixdb
+inventory=$temp/inventory
+conf=$temp/cc.conf
+java -cp "../repo/*" org.apache.asterixdb.aws.ConfigGenerator $temp/nodes $inventory $conf
+
+# wait a while so that all instances are up and running
+# TODO(yingyi) pull the "status check" field of each instance.
+sleep 60
+
+# install asterixdb on all AWS instances
+export ANSIBLE_HOST_KEY_CHECKING=false
+ansible-playbook -i $inventory ansible/instance_start.yml
+
diff --git a/asterixdb/asterix-server/src/main/aws/bin/stop.sh b/asterixdb/asterix-server/src/main/aws/bin/stop.sh
new file mode 100755
index 0000000..44d5a3c
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/bin/stop.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -x
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+
+# starts an AWS cluster
+ansible-playbook -i "localhost," ansible/aws_stop.yml
diff --git a/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml b/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
new file mode 100644
index 0000000..4d62667
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
@@ -0,0 +1,46 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+# The OS image id for ec2 instances.
+image: ami-dce3b1bc
+
+# The data center region for ec2 instances.
+region: us-west-1
+
+# The tag for each ec2 machine.
+tag: scale_test
+
+# The name of a security group that appears in your AWS console.
+group: default
+
+# The name of a key pair that appears in your AWS console.
+keypair: <to be filled>
+
+# The AWS access key id for your IAM user.
+access_key_id: <to be filled>
+
+# The AWS secrety key for your IAM user.
+secret_access_key: <to be filled>
+
+# The AWS instance type. A full list of available types are listed at:
+# https://aws.amazon.com/ec2/instance-types/
+instance_type: t2.micro
+
+# The number of ec2 instances that construct a cluster.
+count: 2
diff --git a/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml b/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
new file mode 100644
index 0000000..1008105
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
@@ -0,0 +1,45 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+# The name of the product (or extension) being used.
+product: asterixdb
+
+# The server binary zip.
+binary: asterix-server-*-binary-assembly.zip
+
+# The script that starts a nc service.
+ncsbin: "asterixncservice"
+
+# The script that starts a cc.
+ccbin: "asterixcc"
+
+# The parent directory for the working directory.
+basedir: /home/ec2-user
+
+# The working directory.
+binarydir: "{{ basedir }}/{{ product }}"
+
+# The pattern for retrieving the sever binary zip from the current build.
+srcpattern: "../../../{{ binary }}"
+
+# The nc service command (script).
+ncservice: "{{ binarydir}}/bin/{{ ncsbin }}"
+
+# The cc service command (script).
+cc: "{{ binarydir}}/bin/{{ ccbin }}"
diff --git a/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
new file mode 100644
index 0000000..8f66e08
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterixdb.aws;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class AwsNode {
+
+ private final String publicIp;
+ private final String privateIp;
+ private final String dnsName;
+
+ public AwsNode(ObjectNode node) {
+ this.publicIp = node.get("public_ip").asText();
+ this.privateIp = node.get("private_ip").asText();
+ this.dnsName = node.get("dns_name").asText();
+ }
+
+ public String getPublicIp() {
+ return publicIp;
+ }
+
+ public String getPrivateIp() {
+ return privateIp;
+ }
+
+ public String getDnsName() {
+ return dnsName;
+ }
+}
diff --git a/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
new file mode 100644
index 0000000..6b6ee54
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterixdb.aws;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class ConfigGenerator {
+
+ public static void main(String[] args) throws Exception {
+ if (args.length < 3) {
+ System.out.println("The usage of ConfigGenerator: ");
+ System.out.println("<input node json file> <output inventory file> <output config file>");
+ System.exit(0);
+ }
+ String source = args[0];
+ String inventory = args[1];
+ String config = args[2];
+
+ // Read Json file data to String
+ byte[] jsonData = Files.readAllBytes(Paths.get(source));
+
+ // Get a list of cluster nodes
+ ObjectMapper objectMapper = new ObjectMapper();
+ ObjectNode root = (ObjectNode) objectMapper.readTree(jsonData);
+ ArrayNode nodes = (ArrayNode) root.get("tagged_instances");
+ Iterator<JsonNode> nodeIterator = nodes.iterator();
+ List<AwsNode> cluster = new ArrayList<>();
+ while (nodeIterator.hasNext()) {
+ ObjectNode node = (ObjectNode) nodeIterator.next();
+ cluster.add(new AwsNode(node));
+ }
+
+ if (cluster.size() <= 0) {
+ return;
+ }
+
+ // Generates inventory file.
+ generateInventoryFile(cluster, inventory);
+
+ // Generates asterixdb config
+ generateConfig(cluster, config);
+ }
+
+ private static void generateInventoryFile(List<AwsNode> cluster, String inventoryPath) throws IOException {
+ Iterator<AwsNode> nodeIterator = cluster.iterator();
+ try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(inventoryPath)))) {
+ // Prints the cc section
+ writer.println("[cc]");
+ String masterIp = nodeIterator.next().getDnsName();
+ writer.println(masterIp);
+ writer.println();
+
+ // Prints the nc section
+ writer.println("[ncs]");
+ writer.println(masterIp); // There is a NC that co-locates with CC.
+ while (nodeIterator.hasNext()) {
+ writer.println(nodeIterator.next().getDnsName());
+ }
+ writer.println();
+
+ // Prints the user
+ writer.println("[all:vars]");
+ writer.println("ansible_ssh_user=ec2-user");
+ }
+ }
+
+ private static void generateConfig(List<AwsNode> cluster, String configPath) throws IOException {
+ Iterator<AwsNode> nodeIterator = cluster.iterator();
+ try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(configPath)))) {
+ // Prints the cc section
+ writer.println("[cc]");
+ String masterIp = nodeIterator.next().getPrivateIp();
+ writer.println("cluster.address=" + masterIp);
+ writer.println();
+
+ // Prints the nc section
+ writer.println("[nc/1]");
+ writer.println("address=" + masterIp); // There is a NC that co-locates with CC.
+ int ncCounter = 2;
+ while (nodeIterator.hasNext()) {
+ writer.println("[nc/" + ncCounter + "]");
+ writer.println("address=" + nodeIterator.next().getPrivateIp());
+ writer.println();
+ }
+
+ // Prints the nc parameter section.
+ writer.println("[nc]");
+ String rootDirectory = "/home/ec2-user/";
+ writer.println("txnlogdir=" + rootDirectory + "txnlog");
+ writer.println("iodevices=" + rootDirectory + "iodevice");
+ writer.println("command=asterixnc");
+ writer.println("app.class=org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint");
+ writer.println();
+ }
+ }
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Yingyi Bu (Code Review)" <do...@asterixdb.incubator.apache.org>.
Yingyi Bu has submitted this change and it was merged.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Add AWS automation scripts to asterix-server.
- Allows users to customize an AWS-based instance by editing
conf/aws_settings.yml and conf/instance_settings.yml.
- Starts an AWS cluster, install JDK and AsterixDB automatically.
- Let the default value of storage.metadata.memorycomponent.numpages
adjust to the available JVM heap size.
Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1475
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
BAD: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
---
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
M asterixdb/asterix-server/pom.xml
M asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
A asterixdb/asterix-server/src/main/aws/README
A asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
A asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
A asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml
A asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
A asterixdb/asterix-server/src/main/aws/bin/start.sh
A asterixdb/asterix-server/src/main/aws/bin/stop.sh
A asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
A asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
17 files changed, 595 insertions(+), 8 deletions(-)
Approvals:
Till Westmann: Looks good to me, approved
Jenkins: Verified; No violations found; Verified
Objections:
Jenkins: Violations found
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
index 384d759..42fb7c3 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
@@ -104,7 +104,7 @@
"storage.memorycomponent.numcomponents" : 2,
"storage.memorycomponent.numpages" : 8,
"storage.memorycomponent.pagesize" : 131072,
- "storage.metadata.memorycomponent.numpages" : 256,
+ "storage.metadata.memorycomponent.numpages" : 85,
"transaction.log.dirs" : {
"asterix_nc1" : "target/txnLogDir/asterix_nc1",
"asterix_nc2" : "target/txnLogDir/asterix_nc2"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm
index 2476197..75c4d3e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm
@@ -104,7 +104,7 @@
"storage.memorycomponent.numcomponents" : 2,
"storage.memorycomponent.numpages" : 8,
"storage.memorycomponent.pagesize" : 131072,
- "storage.metadata.memorycomponent.numpages" : 256,
+ "storage.metadata.memorycomponent.numpages" : 85,
"transaction.log.dirs" : {
"asterix_nc1" : "target/txnLogDir/asterix_nc1",
"asterix_nc2" : "target/txnLogDir/asterix_nc2"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm
index 5d124e7..76219aa 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm
@@ -104,7 +104,7 @@
"storage.memorycomponent.numcomponents" : 2,
"storage.memorycomponent.numpages" : 8,
"storage.memorycomponent.pagesize" : 131072,
- "storage.metadata.memorycomponent.numpages" : 256,
+ "storage.metadata.memorycomponent.numpages" : 85,
"transaction.log.dirs" : {
"asterix_nc1" : "target/txnLogDir/asterix_nc1",
"asterix_nc2" : "target/txnLogDir/asterix_nc2"
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index faacc64..68ad80c 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -18,10 +18,10 @@
*/
package org.apache.asterix.common.config;
+import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
+
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.util.StorageUtil;
-
-import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
public class StorageProperties extends AbstractProperties {
@@ -40,7 +40,6 @@
private static final String STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY =
"storage.metadata.memorycomponent.numpages";
- private static final int STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 256; // ... so 32MB components
private static final String STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_KEY = "storage.memorycomponent.numcomponents";
private static final int STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_DEFAULT = 2; // 2 components
@@ -53,6 +52,7 @@
private final long storageBufferCacheSizeDefault;
private final int storageMemoryComponentNumPages;
+ private final int storageMetadataMemoryComponentNumPages;
private final long storageMemorycomponentGlobalbudgetDefault;
public StorageProperties(PropertiesAccessor accessor) {
@@ -68,6 +68,10 @@
// for a dataset, including data and indexes.
storageMemoryComponentNumPages = (int) (storageMemorycomponentGlobalbudgetDefault
/ (16 * getMemoryComponentPageSize()));
+ // By default, uses the min of 1/64 of the storageMemorycomponentGlobalbudgetDefault and 256 pages
+ // for the write buffer budget for a metadata dataset, including data and indexes.
+ storageMetadataMemoryComponentNumPages = Math
+ .min((int) (storageMemorycomponentGlobalbudgetDefault / (64 * getMemoryComponentPageSize())), 256);
}
@PropertyKey(STORAGE_BUFFERCACHE_PAGESIZE_KEY)
@@ -107,8 +111,7 @@
@PropertyKey(STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY)
public int getMetadataMemoryComponentNumPages() {
return accessor.getProperty(STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY,
- STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_DEFAULT,
- PropertyInterpreters.getIntegerPropertyInterpreter());
+ storageMetadataMemoryComponentNumPages, PropertyInterpreters.getIntegerPropertyInterpreter());
}
@PropertyKey(STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_KEY)
diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml
index 3454428..f91ed64 100644
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@ -493,5 +493,9 @@
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
index b968ae1..32a0180 100644
--- a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
+++ b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
@@ -52,6 +52,23 @@
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
+ <directory>src/main/aws</directory>
+ <outputDirectory>aws</outputDirectory>
+ <excludes>
+ <exclude>**/*.sh</exclude>
+ </excludes>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/aws</directory>
+ <outputDirectory>aws</outputDirectory>
+ <includes>
+ <include>**/*.sh</include>
+ </includes>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
<directory>target/appassembler/repo</directory>
<outputDirectory>repo</outputDirectory>
</fileSet>
diff --git a/asterixdb/asterix-server/src/main/aws/README b/asterixdb/asterix-server/src/main/aws/README
new file mode 100644
index 0000000..ebd8fee
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/README
@@ -0,0 +1,54 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+To start an AWS cluster, you need to do the following steps:
+
+1. Create an AWS account and an IAM user.
+ Sets up a security group that you'd like to use for your AWS cluster. The security group should at least allow
+ all TCP connection from anywhere.
+
+2. Retrieve your AWS key pair name and fill that after "keypair:" in conf/aws_settings.yml;
+ retrieve your AWS "access key ID" and fill that after "access_key_id:" in conf/aws_settings.yml;
+ retrieve your AWS "secret access key" and fill that after "secret_access_key:" in conf/aws_settings.yml.
+ Note that you can only read or download "access key ID" and "secret access key" once from your AWS console.
+ If you forget them, you have to create new keys again and delete the old ones.
+
+3. Customize other settings in conf/aws_settings.yml and conf/instance_settings.yml to whatever you want.
+ Note that you have to make sure that the security group name (e.g., "group: default") matches the one you setup
+ in step 1.
+
+4. Configure your ssh setting by editing ~/.ssh/config and adding the following entry:
+ Host *.amazonaws.com
+ IdentityFile <path_of_private_key>
+ Note that <path_of_private_key> should be replaced by the path to the file that stores the private key for the key
+ pair that you uploaded to AWS and used in step 2. For example:
+ Host *.amazonaws.com
+ IdentityFile ~/.ssh/id_rsa
+
+5. Install ansible and boto3:
+ ansible: http://docs.ansible.com/ansible/intro_installation.html
+ boto3: pip install boto3
+
+6. Launch your cluster instance on AWS:
+ bin/start.sh
+ Now you can use the AWS-based instance.
+
+7. Terminate the cluster instance on AWS:
+ bin/stop.sh
+ Note that it will destroy the instance and terminate all ec2 instances that you launched in step 6.
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml b/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
new file mode 100644
index 0000000..3deab82
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
@@ -0,0 +1,48 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Start AWS cluster
+ hosts: localhost
+ gather_facts: false
+ tasks:
+ - include_vars: ../conf/aws_settings.yml
+ - name: Launch all instances
+ ec2:
+ key_name: "{{ keypair }}"
+ group: "{{ group }}"
+ instance_type: "{{ instance_type }}"
+ image: "{{ image }}"
+ wait: true
+ region: "{{ region }}"
+ aws_access_key: "{{ access_key_id }}"
+ aws_secret_key: "{{ secret_access_key }}"
+ exact_count: "{{ count }}"
+ count_tag:
+ Name: "{{ tag }}"
+ instance_tags:
+ Name: "{{ tag }}"
+ register: ec2
+ - name: Create local temporary directory
+ file:
+ path: /tmp/asterixdb
+ state: directory
+ mode: 0755
+ - name: Output the information of all nodes
+ local_action: copy content="{{ ec2|to_json }}" dest=/tmp/asterixdb/nodes
+
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml b/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
new file mode 100644
index 0000000..e1bdc86
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
@@ -0,0 +1,35 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Stop AWS cluster
+ hosts: localhost
+ gather_facts: false
+ tasks:
+ - include_vars: ../conf/aws_settings.yml
+ - name: Stop all instance
+ ec2:
+ key_name: "{{ keypair }}"
+ group: "{{ group }}"
+ instance_type: "{{ instance_type }}"
+ image: "{{ image }}"
+ wait: true
+ region: "{{ region }}"
+ aws_access_key: "{{ access_key_id }}"
+ aws_secret_key: "{{ secret_access_key }}"
+ exact_count: 0
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml b/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml
new file mode 100644
index 0000000..bb69b68
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml
@@ -0,0 +1,28 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Ensure the working directory exists
+ file:
+ path: "{{ binarydir }}"
+ state: directory
+
+- name: Unzip binary
+ unarchive:
+ src: "{{ lookup('pipe', 'ls -1 ' + srcpattern) }}"
+ dest: "{{ binarydir }}"
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml b/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
new file mode 100644
index 0000000..743b70e
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
@@ -0,0 +1,63 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- hosts: ncs
+ tasks:
+ - include_vars: ../conf/instance_settings.yml
+ - name: Download JDK
+ shell: "wget -q --tries=5 --no-cookies --no-check-certificate --header \
+ \"Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie\" \
+ \"http://download.oracle.com/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-x64.rpm\""
+
+ - name: Install JDK
+ shell: sudo yum -y localinstall jdk-8u121-linux-x64.rpm
+
+ - include: customize_deployment.yml
+
+ - name: Ensure the log directory exists
+ file:
+ path: "{{ binarydir }}/logs"
+ state: directory
+
+ - name: Ensure the io device directory exit
+ file:
+ path: "{{ basedir }}/iodevice"
+ state: directory
+
+ - name: Start NC Service
+ shell: nohup "{{ ncservice }}" &> "{{ binarydir }}/logs/ncservice.log" &
+ async: 10
+ poll: 0
+
+- hosts: cc
+ tasks:
+ - include_vars: ../conf/instance_settings.yml
+ - name: Copy cluster config to CC
+ copy:
+ src: /tmp/asterixdb/cc.conf
+ dest: "{{ basedir }}/cc.conf"
+
+ - name: Update cluster config
+ shell: find -P "{{ basedir }}/cc.conf"|xargs perl -pi -e 's|command=asterixnc|command={{ ncbin }}|g'
+
+ - name: Start CC
+ shell: nohup "{{ cc }}" -config-file "{{ basedir }}/cc.conf" &> "{{ binarydir }}/logs/cc.log" &
+ async: 10
+ poll: 0
+
diff --git a/asterixdb/asterix-server/src/main/aws/bin/start.sh b/asterixdb/asterix-server/src/main/aws/bin/start.sh
new file mode 100755
index 0000000..3779952
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/bin/start.sh
@@ -0,0 +1,38 @@
+#!/bin/bash -x
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+
+# Starts an AWS cluster.
+ansible-playbook -i "localhost," ansible/aws_start.yml
+
+# Generates an Ansible inventory file and an AsterixDB configuration file.
+temp=/tmp/asterixdb
+inventory=$temp/inventory
+conf=$temp/cc.conf
+java -cp "../repo/*" org.apache.asterixdb.aws.ConfigGenerator $temp/nodes $inventory $conf
+
+# Waits a while so that all instances are up and running.
+# TODO(yingyi) pull the "status check" field of each instance.
+sleep 90
+
+# Installs asterixdb on all AWS instances.
+export ANSIBLE_HOST_KEY_CHECKING=false
+ansible-playbook -i $inventory ansible/instance_start.yml
+
diff --git a/asterixdb/asterix-server/src/main/aws/bin/stop.sh b/asterixdb/asterix-server/src/main/aws/bin/stop.sh
new file mode 100755
index 0000000..fd2c85b
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/bin/stop.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -x
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+
+# Terminates an AWS cluster
+ansible-playbook -i "localhost," ansible/aws_stop.yml
diff --git a/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml b/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
new file mode 100644
index 0000000..06a7d27
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
@@ -0,0 +1,46 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+# The OS image id for ec2 instances.
+image: ami-76fa4116
+
+# The data center region for ec2 instances.
+region: us-west-2
+
+# The tag for each ec2 machine.
+tag: scale_test
+
+# The name of a security group that appears in your AWS console.
+group: default
+
+# The name of a key pair that appears in your AWS console.
+keypair: <to be filled>
+
+# The AWS access key id for your IAM user.
+access_key_id: <to be filled>
+
+# The AWS secrety key for your IAM user.
+secret_access_key: <to be filled>
+
+# The AWS instance type. A full list of available types are listed at:
+# https://aws.amazon.com/ec2/instance-types/
+instance_type: t2.micro
+
+# The number of ec2 instances that construct a cluster.
+count: 2
diff --git a/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml b/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
new file mode 100644
index 0000000..de83b7c
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
@@ -0,0 +1,48 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+# The name of the product (or extension) being used.
+product: asterixdb
+
+# The server binary zip.
+binary: asterix-server-*-binary-assembly.zip
+
+# The script that starts a nc service.
+ncsbin: "asterixncservice"
+
+# The script that starts a nc.
+ncbin: "asterixnc"
+
+# The script that starts a cc.
+ccbin: "asterixcc"
+
+# The parent directory for the working directory.
+basedir: /home/ec2-user
+
+# The working directory.
+binarydir: "{{ basedir }}/{{ product }}"
+
+# The pattern for retrieving the sever binary zip from the current build.
+srcpattern: "../../../{{ binary }}"
+
+# The nc service command (script).
+ncservice: "{{ binarydir}}/bin/{{ ncsbin }}"
+
+# The cc service command (script).
+cc: "{{ binarydir}}/bin/{{ ccbin }}"
diff --git a/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
new file mode 100644
index 0000000..fb45643
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterixdb.aws;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+/**
+ * The useful information for an AWS node.
+ */
+public class AwsNode {
+
+ private final String privateIp;
+ private final String dnsName;
+
+ public AwsNode(ObjectNode node) {
+ this.privateIp = node.get("private_ip").asText();
+ this.dnsName = node.get("dns_name").asText();
+ }
+
+ public String getPrivateIp() {
+ return privateIp;
+ }
+
+ public String getDnsName() {
+ return dnsName;
+ }
+}
diff --git a/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
new file mode 100644
index 0000000..25895e8
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterixdb.aws;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+/**
+ * This class is the binary that automatically generates :
+ * 1. an Ansible inventory file for the AWS cluster;
+ * 2. an AsterixDB configuration file for the AWS cluster
+ * from a JSON description file returned by the Ansible AWS startup script.
+ */
+public class ConfigGenerator {
+
+ private ConfigGenerator() {
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length < 3) {
+ System.err.println("The usage of ConfigGenerator: ");
+ System.err.println("<input node json file> <output inventory file> <output config file>");
+ System.exit(0);
+ }
+ String source = args[0];
+ String inventory = args[1];
+ String config = args[2];
+
+ // Read Json file data to String
+ byte[] jsonData = Files.readAllBytes(Paths.get(source));
+
+ // Get a list of cluster nodes
+ ObjectMapper objectMapper = new ObjectMapper();
+ ObjectNode root = (ObjectNode) objectMapper.readTree(jsonData);
+ ArrayNode nodes = (ArrayNode) root.get("tagged_instances");
+ Iterator<JsonNode> nodeIterator = nodes.iterator();
+ List<AwsNode> cluster = new ArrayList<>();
+ while (nodeIterator.hasNext()) {
+ ObjectNode node = (ObjectNode) nodeIterator.next();
+ cluster.add(new AwsNode(node));
+ }
+
+ if (cluster.isEmpty()) {
+ return;
+ }
+
+ // Generates inventory file.
+ generateInventoryFile(cluster, inventory);
+
+ // Generates asterixdb config
+ generateConfig(cluster, config);
+ }
+
+ private static void generateInventoryFile(List<AwsNode> cluster, String inventoryPath) throws IOException {
+ Iterator<AwsNode> nodeIterator = cluster.iterator();
+ try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(inventoryPath)))) {
+ // Prints the cc section
+ writer.println("[cc]");
+ String masterIp = nodeIterator.next().getDnsName();
+ writer.println(masterIp);
+ writer.println();
+
+ // Prints the nc section
+ writer.println("[ncs]");
+ writer.println(masterIp); // There is a NC that co-locates with CC.
+ while (nodeIterator.hasNext()) {
+ writer.println(nodeIterator.next().getDnsName());
+ }
+ writer.println();
+
+ // Prints the user
+ writer.println("[all:vars]");
+ writer.println("ansible_ssh_user=ec2-user");
+ }
+ }
+
+ private static void generateConfig(List<AwsNode> cluster, String configPath) throws IOException {
+ Iterator<AwsNode> nodeIterator = cluster.iterator();
+ try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(configPath)))) {
+ // Prints the cc section
+ writer.println("[cc]");
+ String masterIp = nodeIterator.next().getPrivateIp();
+ writer.println("cluster.address=" + masterIp);
+ writer.println();
+
+ // Prints the nc section
+ writer.println("[nc/1]");
+ writer.println("address=" + masterIp); // There is a NC that co-locates with CC.
+ int ncCounter = 2;
+ while (nodeIterator.hasNext()) {
+ writer.println("[nc/" + ncCounter++ + "]");
+ writer.println("address=" + nodeIterator.next().getPrivateIp());
+ writer.println();
+ }
+
+ // Prints the nc parameter section.
+ writer.println("[nc]");
+ String rootDirectory = "/home/ec2-user/";
+ writer.println("txnlogdir=" + rootDirectory + "txnlog");
+
+ // TODO(yingyi): figure out how to get device mapping for SSD-based instances.
+ writer.println("iodevices=" + rootDirectory + "iodevice");
+ writer.println("command=asterixnc");
+ writer.println();
+ }
+ }
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 10
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 9:
Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1755/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 8:
BAD Compatibility Tests Started https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/420/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 7:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4110/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 6: BAD+1
BAD Compatibility Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/415/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 9: Integration-Tests+1
Integration Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1755/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 8:
Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1750/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Yingyi Bu (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/1475
to look at the new patch set (#4).
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Add AWS automation scripts to asterix-server.
- Allows users to customize an AWS-based instance by editing
conf/aws_settings.yml and conf/instance_settings.yml.
- Starts an AWS cluster, install JDK and AsterixDB automatically.
- Let the default value of storage.metadata.memorycomponent.numpages
adjust to the available JVM heap size.
Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
---
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
M asterixdb/asterix-server/pom.xml
M asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
A asterixdb/asterix-server/src/main/aws/README
A asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
A asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
A asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
A asterixdb/asterix-server/src/main/aws/bin/start.sh
A asterixdb/asterix-server/src/main/aws/bin/stop.sh
A asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
A asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
13 files changed, 559 insertions(+), 5 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/75/1475/4
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 9:
Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1752/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Yingyi Bu (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/1475
to look at the new patch set (#8).
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Add AWS automation scripts to asterix-server.
- Allows users to customize an AWS-based instance by editing
conf/aws_settings.yml and conf/instance_settings.yml.
- Starts an AWS cluster, install JDK and AsterixDB automatically.
- Let the default value of storage.metadata.memorycomponent.numpages
adjust to the available JVM heap size.
Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
---
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
M asterixdb/asterix-server/pom.xml
M asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
A asterixdb/asterix-server/src/main/aws/README
A asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
A asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
A asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml
A asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
A asterixdb/asterix-server/src/main/aws/bin/start.sh
A asterixdb/asterix-server/src/main/aws/bin/stop.sh
A asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
A asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
17 files changed, 595 insertions(+), 8 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/75/1475/8
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Yingyi Bu (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/1475
to look at the new patch set (#7).
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Add AWS automation scripts to asterix-server.
- Allows users to customize an AWS-based instance by editing
conf/aws_settings.yml and conf/instance_settings.yml.
- Starts an AWS cluster, install JDK and AsterixDB automatically.
- Let the default value of storage.metadata.memorycomponent.numpages
adjust to the available JVM heap size.
Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
---
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
M asterixdb/asterix-server/pom.xml
M asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
A asterixdb/asterix-server/src/main/aws/README
A asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
A asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
A asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml
A asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
A asterixdb/asterix-server/src/main/aws/bin/start.sh
A asterixdb/asterix-server/src/main/aws/bin/stop.sh
A asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
A asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
17 files changed, 587 insertions(+), 8 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/75/1475/7
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 6:
BAD Compatibility Tests Started https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/415/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 4:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4104/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 4
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Till Westmann (Code Review)" <do...@asterixdb.incubator.apache.org>.
Till Westmann has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 7:
(3 comments)
Just a few small comments so far. One bigger question I have if this should be a separate module. Maybe we can discuss this on the list? In any case this decision can be taken late on as well, so we don't need to block progress on this.
https://asterix-gerrit.ics.uci.edu/#/c/1475/7/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
File asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml:
PS7, Line 26: 8u60
8u60 seems old - 8u121 is the current version.
PS7, Line 29: jre
Is a JRE enough? Just curious - it should be, but I've never tried that.
https://asterix-gerrit.ics.uci.edu/#/c/1475/7/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
File asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml:
PS7, Line 24: us-west-1
Please don't use us-west-1 (N. California) as the default as it is quite expensive. us-west-2 (Oregon) seems more reasonable [1].
[1] https://www.concurrencylabs.com/blog/choose-your-aws-region-wisely/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: Yes
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 7: Integration-Tests+1
Integration Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1747/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 1:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4101/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 9:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4114/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Yingyi Bu (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/1475
to look at the new patch set (#9).
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Add AWS automation scripts to asterix-server.
- Allows users to customize an AWS-based instance by editing
conf/aws_settings.yml and conf/instance_settings.yml.
- Starts an AWS cluster, install JDK and AsterixDB automatically.
- Let the default value of storage.metadata.memorycomponent.numpages
adjust to the available JVM heap size.
Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
---
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
M asterixdb/asterix-server/pom.xml
M asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
A asterixdb/asterix-server/src/main/aws/README
A asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
A asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
A asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml
A asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
A asterixdb/asterix-server/src/main/aws/bin/start.sh
A asterixdb/asterix-server/src/main/aws/bin/stop.sh
A asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
A asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
17 files changed, 595 insertions(+), 8 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/75/1475/9
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Yingyi Bu (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/1475
to look at the new patch set (#3).
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Add AWS automation scripts to asterix-server.
- Allows users to customize an AWS-based instance by editing
conf/aws_settings.yml and conf/instance_settings.yml.
- Starts an AWS cluster, install JDK and AsterixDB automatically.
- Let the default value of storage.metadata.memorycomponent.numpages
adjust to the available JVM heap size.
Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
---
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
M asterixdb/asterix-server/pom.xml
M asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
A asterixdb/asterix-server/src/main/aws/README
A asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
A asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
A asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
A asterixdb/asterix-server/src/main/aws/bin/start.sh
A asterixdb/asterix-server/src/main/aws/bin/stop.sh
A asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
A asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
13 files changed, 556 insertions(+), 5 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/75/1475/3
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Yingyi Bu (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/1475
to look at the new patch set (#6).
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Add AWS automation scripts to asterix-server.
- Allows users to customize an AWS-based instance by editing
conf/aws_settings.yml and conf/instance_settings.yml.
- Starts an AWS cluster, install JDK and AsterixDB automatically.
- Let the default value of storage.metadata.memorycomponent.numpages
adjust to the available JVM heap size.
Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
---
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
M asterixdb/asterix-server/pom.xml
M asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
A asterixdb/asterix-server/src/main/aws/README
A asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
A asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
A asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml
A asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
A asterixdb/asterix-server/src/main/aws/bin/start.sh
A asterixdb/asterix-server/src/main/aws/bin/stop.sh
A asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
A asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
17 files changed, 586 insertions(+), 8 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/75/1475/6
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 9: Integration-Tests-1
Integration Tests Timed Out
https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1752/ : ABORTED
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Yingyi Bu (Code Review)" <do...@asterixdb.incubator.apache.org>.
Yingyi Bu has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 7:
(3 comments)
https://asterix-gerrit.ics.uci.edu/#/c/1475/7/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
File asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml:
PS7, Line 26: 8u60
> 8u60 seems old - 8u121 is the current version.
Done
PS7, Line 29: jre
> Is a JRE enough? Just curious - it should be, but I've never tried that.
Done
https://asterix-gerrit.ics.uci.edu/#/c/1475/7/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
File asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml:
PS7, Line 24: us-west-1
> Please don't use us-west-1 (N. California) as the default as it is quite ex
Done
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: Yes
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 8:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4113/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 8: BAD+1
BAD Compatibility Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/420/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 8
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 9: BAD+1
BAD Compatibility Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/421/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 9:
BAD Compatibility Tests Started https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/421/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 7: BAD+1
BAD Compatibility Tests Successful
https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/417/ : SUCCESS
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 3:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4103/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 2:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4102/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 6:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4108/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Yingyi Bu (Code Review)" <do...@asterixdb.incubator.apache.org>.
Yingyi Bu has uploaded a new patch set (#2).
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Add AWS automation scripts to asterix-server.
- Allows users to customize an AWS-based instance by editing
conf/aws_settings.yml and conf/instance_settings.yml.
- Starts an AWS cluster, install JDK and AsterixDB automatically.
- Let the default value of storage.metadata.memorycomponent.numpages
adjust to the available JVM heap size.
Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
---
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
M asterixdb/asterix-server/pom.xml
M asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
A asterixdb/asterix-server/src/main/aws/README
A asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
A asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
A asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
A asterixdb/asterix-server/src/main/aws/bin/start.sh
A asterixdb/asterix-server/src/main/aws/bin/stop.sh
A asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
A asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
13 files changed, 556 insertions(+), 5 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/75/1475/2
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 6:
Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1745/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Till Westmann (Code Review)" <do...@asterixdb.incubator.apache.org>.
Till Westmann has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 9: Code-Review+2
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 9
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <bu...@gmail.com>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Yingyi Bu (Code Review)" <do...@asterixdb.incubator.apache.org>.
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/1475
to look at the new patch set (#5).
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Add AWS automation scripts to asterix-server.
- Allows users to customize an AWS-based instance by editing
conf/aws_settings.yml and conf/instance_settings.yml.
- Starts an AWS cluster, install JDK and AsterixDB automatically.
- Let the default value of storage.metadata.memorycomponent.numpages
adjust to the available JVM heap size.
Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
---
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
M asterixdb/asterix-server/pom.xml
M asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
A asterixdb/asterix-server/src/main/aws/README
A asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
A asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
A asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
A asterixdb/asterix-server/src/main/aws/bin/start.sh
A asterixdb/asterix-server/src/main/aws/bin/stop.sh
A asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
A asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
A asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
16 files changed, 562 insertions(+), 8 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/75/1475/5
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 5
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 7:
BAD Compatibility Tests Started https://asterix-jenkins.ics.uci.edu/job/asterixbad-compat/417/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 7:
Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/1747/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: No
Change in asterixdb[master]: Add AWS automation scripts to asterix-server.
Posted by "Jenkins (Code Review)" <do...@asterixdb.incubator.apache.org>.
Jenkins has posted comments on this change.
Change subject: Add AWS automation scripts to asterix-server.
......................................................................
Patch Set 5:
Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/4105/
--
To view, visit https://asterix-gerrit.ics.uci.edu/1475
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: comment
Gerrit-Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Gerrit-PatchSet: 5
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <bu...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Michael Blow <mb...@apache.org>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-HasComments: No