You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by pt...@apache.org on 2016/09/14 10:53:28 UTC
[23/35] ignite git commit: IGNITE-3172 Refactoring Ignite-Cassandra
serializers. - Fixes #956.
http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-env.sh
----------------------------------------------------------------------
diff --git a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-env.sh b/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-env.sh
deleted file mode 100644
index bfe3371..0000000
--- a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-env.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/sh
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# -----------------------------------------------------------------------------------------------
-# Environment setup script from Ignite
-# -----------------------------------------------------------------------------------------------
-
-JVM_OPTS="-Xms10g -Xmx10g -server -XX:+AggressiveOpts -XX:MaxMetaspaceSize=256m"
-JVM_OPTS="$JVM_OPTS -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+UseTLAB -XX:NewSize=128m -XX:MaxNewSize=768m"
-#JVM_OPTS="$JVM_OPTS -XX:MaxTenuringThreshold=0 -XX:SurvivorRatio=1024 -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=60"
-JVM_OPTS="$JVM_OPTS -Xss16m"
-
-export JVM_OPTS
http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-start.sh
----------------------------------------------------------------------
diff --git a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-start.sh b/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-start.sh
deleted file mode 100644
index f2c1557..0000000
--- a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-start.sh
+++ /dev/null
@@ -1,266 +0,0 @@
-#!/bin/sh
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# -----------------------------------------------------------------------------------------------
-# Script to start Ignite daemon (used by ignite-bootstrap.sh)
-# -----------------------------------------------------------------------------------------------
-
-#profile=/home/ignite/.bash_profile
-profile=/root/.bash_profile
-
-. $profile
-. /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "ignite"
-
-# Setups Cassandra seeds for this Ignite node being able to connect to Cassandra.
-# Looks for the information in S3 about already up and running Cassandra cluster nodes.
-setupCassandraSeeds()
-{
- setupClusterSeeds "cassandra" "true"
-
- CLUSTER_SEEDS=($CLUSTER_SEEDS)
- count=${#CLUSTER_SEEDS[@]}
-
- CASSANDRA_SEEDS=
-
- for (( i=0; i<=$(( $count -1 )); i++ ))
- do
- seed=${CLUSTER_SEEDS[$i]}
- CASSANDRA_SEEDS="${CASSANDRA_SEEDS}<value>$seed<\/value>"
- done
-
- cat /opt/ignite/config/ignite-cassandra-server-template.xml | sed -r "s/\\\$\{CASSANDRA_SEEDS\}/$CASSANDRA_SEEDS/g" > /opt/ignite/config/ignite-cassandra-server.xml
-}
-
-# Setups Ignite nodes which this EC2 Ignite node will use to send its metadata and join Ignite cluster
-setupIgniteSeeds()
-{
- if [ "$FIRST_NODE_LOCK" == "true" ]; then
- echo "[INFO] Setting up Ignite seeds"
-
- CLUSTER_SEEDS="127.0.0.1:47500..47509"
-
- echo "[INFO] Using localhost address as a seed for the first Ignite node: $CLUSTER_SEEDS"
-
- aws s3 rm --recursive ${S3_IGNITE_NODES_DISCOVERY::-1}
- if [ $? -ne 0 ]; then
- terminate "Failed to clean Ignite node discovery URL: $S3_IGNITE_NODES_DISCOVERY"
- fi
- else
- setupClusterSeeds "ignite" "true"
- fi
-
- CLUSTER_SEEDS=($CLUSTER_SEEDS)
- count=${#CLUSTER_SEEDS[@]}
-
- IGNITE_SEEDS=
-
- for (( i=0; i<=$(( $count -1 )); i++ ))
- do
- seed=${CLUSTER_SEEDS[$i]}
- IGNITE_SEEDS="${IGNITE_SEEDS}<value>$seed<\/value>"
- done
-
- cat /opt/ignite/config/ignite-cassandra-server.xml | sed -r "s/\\\$\{IGNITE_SEEDS\}/$IGNITE_SEEDS/g" > /opt/ignite/config/ignite-cassandra-server1.xml
- mv -f /opt/ignite/config/ignite-cassandra-server1.xml /opt/ignite/config/ignite-cassandra-server.xml
-}
-
-# Checks status of Ignite daemon
-checkIgniteStatus()
-{
- proc=$(ps -ef | grep java | grep "org.apache.ignite.startup.cmdline.CommandLineStartup")
-
- nodeId=
- nodeAddrs=
- nodePorts=
- topology=
- metrics=
-
- logFile=$(ls /opt/ignite/work/log/ | grep "\.log$")
- if [ -n "$logFile" ]; then
- logFile=/opt/ignite/work/log/$logFile
- nodeId=$(cat $logFile | grep "Local node \[ID")
- nodeAddrs=$(cat $logFile | grep "Local node addresses:")
- nodePorts=$(cat $logFile | grep "Local ports:")
- topology=$(cat $logFile | grep "Topology snapshot")
- metrics=$(cat $logFile | grep "Metrics for local node" | head -n 1)
- fi
-
- if [ -n "$nodeId" ] && [ -n "$nodeAddrs" ] && [ -n "$nodePorts" ] && [ -n "$topology" ] && [ -n "$metrics" ] && [ -n "$proc" ]; then
- sleep 30s
- return 0
- fi
-
- return 1
-}
-
-# Gracefully starts Ignite daemon and waits until it joins Ignite cluster
-startIgnite()
-{
- echo "[INFO]-------------------------------------------------------------"
- echo "[INFO] Trying attempt $START_ATTEMPT to start Ignite daemon"
- echo "[INFO]-------------------------------------------------------------"
- echo ""
-
- setupCassandraSeeds
- setupIgniteSeeds
-
- waitToJoinCluster
-
- if [ "$FIRST_NODE_LOCK" == "true" ]; then
- aws s3 rm --recursive ${S3_IGNITE_NODES_DISCOVERY::-1}
- if [ $? -ne 0 ]; then
- terminate "Failed to clean Ignite node discovery URL: $S3_IGNITE_NODES_DISCOVERY"
- fi
- fi
-
- proc=$(ps -ef | grep java | grep "org.apache.ignite.startup.cmdline.CommandLineStartup")
- proc=($proc)
-
- if [ -n "${proc[1]}" ]; then
- echo "[INFO] Terminating existing Ignite process ${proc[1]}"
- kill -9 ${proc[1]}
- fi
-
- echo "[INFO] Starting Ignite"
- rm -Rf /opt/ignite/work/*
- /opt/ignite/bin/ignite.sh /opt/ignite/config/ignite-cassandra-server.xml &
-
- echo "[INFO] Ignite job id: $!"
-
- sleep 1m
-
- START_ATTEMPT=$(( $START_ATTEMPT+1 ))
-}
-
-#######################################################################################################
-
-START_ATTEMPT=0
-
-# Cleans all the previous metadata about this EC2 node
-unregisterNode
-
-# Tries to get first-node lock
-tryToGetFirstNodeLock
-
-echo "[INFO]-----------------------------------------------------------------"
-
-if [ "$FIRST_NODE_LOCK" == "true" ]; then
- echo "[INFO] Starting first Ignite node"
-else
- echo "[INFO] Starting Ignite node"
-fi
-
-echo "[INFO]-----------------------------------------------------------------"
-printInstanceInfo
-echo "[INFO]-----------------------------------------------------------------"
-
-if [ "$FIRST_NODE_LOCK" != "true" ]; then
- waitFirstClusterNodeRegistered "true"
-else
- cleanupMetadata
-fi
-
-# Applies Ignite environment settings from ignite-env.sh
-envScript=$(readlink -m $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/ignite-env.sh)
-if [ -f "$envScript" ]; then
- . $envScript
-fi
-
-# Start Ignite daemon
-startIgnite
-
-startTime=$(date +%s)
-
-# Trying multiple attempts to start Ignite daemon
-while true; do
- proc=$(ps -ef | grep java | grep "org.apache.ignite.startup.cmdline.CommandLineStartup")
-
- checkIgniteStatus
-
- if [ $? -eq 0 ]; then
- sleep 1m
- echo "[INFO]-----------------------------------------------------"
- echo "[INFO] Ignite daemon successfully started"
- echo "[INFO]-----------------------------------------------------"
- echo $proc
- echo "[INFO]-----------------------------------------------------"
-
- # Once node joined the cluster we need to remove cluster-join lock
- # to allow other EC2 nodes to acquire it and join cluster sequentially
- removeClusterJoinLock
-
- break
- fi
-
- currentTime=$(date +%s)
- duration=$(( $currentTime-$startTime ))
- duration=$(( $duration/60 ))
-
- if [ $duration -gt $SERVICE_STARTUP_TIME ]; then
- if [ "$FIRST_NODE_LOCK" == "true" ]; then
- # If the first node of Ignite cluster failed to start Ignite daemon in SERVICE_STARTUP_TIME min,
- # we will not try any other attempts and just terminate with error. Terminate function itself, will
- # take care about removing all the locks holding by this node.
- terminate "${SERVICE_STARTUP_TIME}min timeout expired, but first Ignite daemon is still not up and running"
- else
- # If node isn't the first node of Ignite cluster and it failed to start we need to
- # remove cluster-join lock to allow other EC2 nodes to acquire it
- removeClusterJoinLock
-
- # If node failed all SERVICE_START_ATTEMPTS attempts to start Ignite daemon we will not
- # try anymore and terminate with error
- if [ $START_ATTEMPT -gt $SERVICE_START_ATTEMPTS ]; then
- terminate "${SERVICE_START_ATTEMPTS} attempts exceed, but Ignite daemon is still not up and running"
- fi
-
- # New attempt to start Ignite daemon
- startIgnite
- fi
-
- continue
- fi
-
- # Handling situation when Ignite daemon process abnormally terminated
- if [ -z "$proc" ]; then
- # If this is the first node of Ignite cluster just terminating with error
- if [ "$FIRST_NODE_LOCK" == "true" ]; then
- terminate "Failed to start Ignite daemon"
- fi
-
- # Remove cluster-join lock to allow other EC2 nodes to acquire it
- removeClusterJoinLock
-
- echo "[WARN] Failed to start Ignite daemon. Sleeping for extra 30sec"
- sleep 30s
-
- # New attempt to start Ignite daemon
- startIgnite
-
- continue
- fi
-
- echo "[INFO] Waiting for Ignite daemon to start, time passed ${duration}min"
- sleep 30s
-done
-
-# Once Ignite daemon successfully started we registering new Ignite node in S3
-registerNode
-
-# Terminating script with zero exit code
-terminate
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/logs-collector.sh
----------------------------------------------------------------------
diff --git a/modules/cassandra/src/test/bootstrap/aws/logs-collector.sh b/modules/cassandra/src/test/bootstrap/aws/logs-collector.sh
deleted file mode 100644
index 1634b89..0000000
--- a/modules/cassandra/src/test/bootstrap/aws/logs-collector.sh
+++ /dev/null
@@ -1,173 +0,0 @@
-#!/bin/sh
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# -----------------------------------------------------------------------------------------------
-# Logs collector daemon
-# -----------------------------------------------------------------------------------------------
-# Script is launched in background by all EC2 nodes of all clusters (Cassandra, Ignite, Tests) and
-# periodically (each 30 seconds) checks if specific S3 trigger file (specified by $S3_LOGS_TRIGGER_URL)
-# was created or its timestamp was changed. Such an event serve as a trigger for the script
-# to collect EC2 instance logs (from folder specified by $1) and upload them into specific
-# S3 folder (specified by $S3_LOGS_FOLDER).
-# -----------------------------------------------------------------------------------------------
-
-uploadLogs()
-{
- if [ ! -d "$1" ]; then
- echo "[INFO] Logs directory doesn't exist: $1"
- return 0
- fi
-
- echo "[INFO] Uploading logs from directory: $1"
-
- dirList=$(ls $1 | head -1)
-
- if [ -z "$dirList" ]; then
- echo "[INFO] Directory is empty: $1"
- fi
-
- for i in 0 9;
- do
- aws s3 sync --sse AES256 --delete "$1" "$S3_LOGS_FOLDER"
- code=$?
-
- if [ $code -eq 0 ]; then
- echo "[INFO] Successfully uploaded logs from directory: $1"
- return 0
- fi
-
- echo "[WARN] Failed to upload logs from $i attempt, sleeping extra 30sec"
- sleep 30s
- done
-
- echo "[ERROR] All 10 attempts to upload logs are failed for the directory: $1"
-}
-
-createNewLogsSnapshot()
-{
- rm -f ~/logs-collector.snapshot.new
-
- for log_src in "$@"
- do
- if [ -d "$log_src" ] || [ -f "$log_src" ]; then
- ls -alR $log_src >> ~/logs-collector.snapshot.new
-
- fi
- done
-}
-
-checkLogsChanged()
-{
- createNewLogsSnapshot $@
-
- if [ ! -f "~/logs-collector.snapshot" ]; then
- return 1
- fi
-
- diff "~/logs-collector.snapshot" "~/logs-collector.snapshot.new" > /dev/null
-
- return $?
-}
-
-updateLogsSnapshot()
-{
- if [ ! -f "~/logs-collector.snapshot.new" ]; then
- return 0
- fi
-
- rm -f "~/logs-collector.snapshot"
- mv "~/logs-collector.snapshot.new" "~/logs-collector.snapshot"
-}
-
-collectLogs()
-{
- createNewLogsSnapshot
-
- rm -Rf ~/logs-collector-logs
- mkdir -p ~/logs-collector-logs
-
- for log_src in "$@"
- do
- if [ -f "$log_src" ]; then
- echo "[INFO] Collecting log file: $log_src"
- cp -f $log_src ~/logs-collector-logs
- elif [ -d "$log_src" ]; then
- echo "[INFO] Collecting logs from folder: $log_src"
- cp -Rf $log_src ~/logs-collector-logs
- fi
- done
-
- uploadLogs ~/logs-collector-logs
-
- rm -Rf ~/logs-collector-logs
-
- updateLogsSnapshot
-}
-
-echo "[INFO] Running Logs collector service"
-
-if [ -z "$1" ]; then
- echo "[ERROR] Logs collection S3 trigger URL doesn't specified"
- exit 1
-fi
-
-S3_LOGS_TRIGGER_URL=$1
-
-echo "[INFO] Logs collection S3 trigger URL: $S3_LOGS_TRIGGER_URL"
-
-if [ -z "$2" ]; then
- echo "[ERROR] S3 folder where to upload logs doesn't specified"
- exit 1
-fi
-
-S3_LOGS_FOLDER=$2
-
-echo "[INFO] S3 logs upload folder: $S3_LOGS_FOLDER"
-
-shift 2
-
-if [ -z "$1" ]; then
- echo "[WARN] Local logs sources don't specified"
-else
- echo "[INFO] Local logs sources: $@"
-fi
-
-echo "--------------------------------------------------------------------"
-
-TRIGGER_STATE=
-
-while true; do
- sleep 30s
-
- STATE=$(aws s3 ls $S3_LOGS_TRIGGER_URL)
-
- if [ -z "$STATE" ] || [ "$STATE" == "$TRIGGER_STATE" ]; then
- checkLogsChanged
-
- if [ $? -eq 0 ]; then
- continue
- fi
- fi
-
- TRIGGER_STATE=$STATE
-
- collectLogs $@ /var/log/cloud-init.log /var/log/cloud-init-output.log
-
- echo "--------------------------------------------------------------------"
-done
http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml
----------------------------------------------------------------------
diff --git a/modules/cassandra/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml b/modules/cassandra/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml
deleted file mode 100644
index 77ffb9e..0000000
--- a/modules/cassandra/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml
+++ /dev/null
@@ -1,192 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<beans xmlns="http://www.springframework.org/schema/beans"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:util="http://www.springframework.org/schema/util"
- xsi:schemaLocation="http://www.springframework.org/schema/beans
- http://www.springframework.org/schema/beans/spring-beans.xsd
- http://www.springframework.org/schema/util
- http://www.springframework.org/schema/util/spring-util.xsd">
-
- <bean id="loadBalancingPolicy" class="com.datastax.driver.core.policies.TokenAwarePolicy">
- <constructor-arg type="com.datastax.driver.core.policies.LoadBalancingPolicy">
- <bean class="com.datastax.driver.core.policies.RoundRobinPolicy"/>
- </constructor-arg>
- </bean>
-
- <util:list id="contactPoints" value-type="java.lang.String">
- ${CASSANDRA_SEEDS}
- </util:list>
-
- <bean id="cassandraAdminDataSource" class="org.apache.ignite.cache.store.cassandra.datasource.DataSource">
- <property name="user" value="cassandra"/>
- <property name="password" value="cassandra"/>
- <property name="contactPoints" ref="contactPoints"/>
- <property name="readConsistency" value="ONE"/>
- <property name="writeConsistency" value="ONE"/>
- <property name="loadBalancingPolicy" ref="loadBalancingPolicy"/>
- </bean>
-
- <!-- Persistence settings for 'cache1' -->
- <bean id="cache1_settings" class="org.apache.ignite.cache.store.cassandra.persistence.KeyValuePersistenceSettings">
- <constructor-arg type="java.lang.String">
- <value><![CDATA[
-<persistence keyspace="test1" table="primitive_test1">
- <keyPersistence class="java.lang.Long" strategy="PRIMITIVE" column="key"/>
- <valuePersistence class="java.lang.Long" strategy="PRIMITIVE" column="value"/>
-</persistence>]]>
- </value>
- </constructor-arg>
- </bean>
-
- <!-- Persistence settings for 'cache2' -->
- <bean id="cache2_settings" class="org.apache.ignite.cache.store.cassandra.persistence.KeyValuePersistenceSettings">
- <constructor-arg type="java.lang.String">
- <value><![CDATA[
-<persistence keyspace="test1" table="blob_test1">
- <keyPersistence class="java.lang.Long" strategy="PRIMITIVE" />
- <valuePersistence strategy="BLOB"/>
-</persistence>]]>
- </value>
- </constructor-arg>
- </bean>
-
- <!-- Persistence settings for 'cache3' -->
- <bean id="cache3_settings" class="org.apache.ignite.cache.store.cassandra.persistence.KeyValuePersistenceSettings">
- <constructor-arg type="java.lang.String">
- <value><![CDATA[
-<persistence keyspace="test1" table="pojo_test3" ttl="86400">
- <keyspaceOptions>
- REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 3}
- AND DURABLE_WRITES = true
- </keyspaceOptions>
- <tableOptions>
- comment = 'A most excellent and useful table'
- AND read_repair_chance = 0.2
- </tableOptions>
- <keyPersistence class="org.apache.ignite.tests.pojos.PersonId" strategy="POJO">
- <partitionKey>
- <field name="companyCode" column="company" />
- <field name="departmentCode" column="department" />
- </partitionKey>
- <clusterKey>
- <field name="personNumber" column="number" sort="desc"/>
- </clusterKey>
- </keyPersistence>
- <valuePersistence class="org.apache.ignite.tests.pojos.Person"
- strategy="POJO"
- serializer="org.apache.ignite.cache.store.cassandra.serializer.KryoSerializer">
- <field name="firstName" column="first_name" />
- <field name="lastName" column="last_name" />
- <field name="age" />
- <field name="married" index="true"/>
- <field name="height" />
- <field name="weight" />
- <field name="birthDate" column="birth_date" />
- <field name="phones" />
- </valuePersistence>
-</persistence>]]>
- </value>
- </constructor-arg>
- </bean>
-
- <!-- Ignite configuration -->
- <bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
- <!-- Turn on client mode -->
- <property name="clientMode" value="true"/>
-
- <property name="metricsLogFrequency" value="0"/>
-
- <property name="cacheConfiguration">
- <list>
- <!-- Configuring persistence for "cache1" cache -->
- <bean class="org.apache.ignite.configuration.CacheConfiguration">
- <property name="name" value="cache1"/>
- <property name="startSize" value="1000000"/>
- <property name="cacheMode" value="PARTITIONED"/>
- <property name="backups" value="0"/>
- <property name="offHeapMaxMemory" value="0"/>
- <property name="swapEnabled" value="false"/>
- <property name="readThrough" value="true"/>
- <property name="writeThrough" value="true"/>
- <property name="writeBehindEnabled" value="true"/>
- <property name="cacheStoreFactory">
- <bean class="org.apache.ignite.cache.store.cassandra.CassandraCacheStoreFactory">
- <property name="dataSourceBean" value="cassandraAdminDataSource"/>
- <property name="persistenceSettingsBean" value="cache1_settings"/>
- </bean>
- </property>
- </bean>
-
- <!-- Configuring persistence for "cache2" cache -->
- <bean class="org.apache.ignite.configuration.CacheConfiguration">
- <property name="name" value="cache2"/>
- <property name="startSize" value="1000000"/>
- <property name="cacheMode" value="PARTITIONED"/>
- <property name="backups" value="0"/>
- <property name="offHeapMaxMemory" value="0"/>
- <property name="swapEnabled" value="false"/>
- <property name="readThrough" value="true"/>
- <property name="writeThrough" value="true"/>
- <property name="writeBehindEnabled" value="true"/>
- <property name="cacheStoreFactory">
- <bean class="org.apache.ignite.cache.store.cassandra.CassandraCacheStoreFactory">
- <property name="dataSourceBean" value="cassandraAdminDataSource"/>
- <property name="persistenceSettingsBean" value="cache2_settings"/>
- </bean>
- </property>
- </bean>
-
- <!-- Configuring persistence for "cache3" cache -->
- <bean class="org.apache.ignite.configuration.CacheConfiguration">
- <property name="name" value="cache3"/>
- <property name="startSize" value="1000000"/>
- <property name="cacheMode" value="PARTITIONED"/>
- <property name="backups" value="0"/>
- <property name="offHeapMaxMemory" value="0"/>
- <property name="swapEnabled" value="false"/>
- <property name="readThrough" value="true"/>
- <property name="writeThrough" value="true"/>
- <property name="writeBehindEnabled" value="true"/>
- <property name="cacheStoreFactory">
- <bean class="org.apache.ignite.cache.store.cassandra.CassandraCacheStoreFactory">
- <property name="dataSourceBean" value="cassandraAdminDataSource"/>
- <property name="persistenceSettingsBean" value="cache3_settings"/>
- </bean>
- </property>
- </bean>
- </list>
- </property>
-
- <!-- Configuring remote ignite cluster connections -->
- <property name="discoverySpi">
- <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
- <property name="ipFinder">
- <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">
- <property name="addresses">
- <list>
- ${IGNITE_SEEDS}
- </list>
- </property>
- </bean>
- </property>
- </bean>
- </property>
- </bean>
-</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/tests/tests-bootstrap.sh
----------------------------------------------------------------------
diff --git a/modules/cassandra/src/test/bootstrap/aws/tests/tests-bootstrap.sh b/modules/cassandra/src/test/bootstrap/aws/tests/tests-bootstrap.sh
deleted file mode 100644
index 8e6faff..0000000
--- a/modules/cassandra/src/test/bootstrap/aws/tests/tests-bootstrap.sh
+++ /dev/null
@@ -1,317 +0,0 @@
-#!/bin/sh
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# -----------------------------------------------------------------------------------------------
-# Bootstrap script to spin up Tests cluster
-# -----------------------------------------------------------------------------------------------
-
-# URL to download AWS CLI tools
-AWS_CLI_DOWNLOAD_URL=https://s3.amazonaws.com/aws-cli/awscli-bundle.zip
-
-# URL to download JDK
-JDK_DOWNLOAD_URL=http://download.oracle.com/otn-pub/java/jdk/8u77-b03/jdk-8u77-linux-x64.tar.gz
-
-# URL to download Ignite-Cassandra tests package - you should previously package and upload it to this place
-TESTS_PACKAGE_DONLOAD_URL=s3://<bucket>/<folder>/ignite-cassandra-tests-<version>.zip
-
-# Terminates script execution and upload logs to S3
-terminate()
-{
- SUCCESS_URL=$S3_TESTS_SUCCESS
- FAILURE_URL=$S3_TESTS_FAILURE
-
- if [ -n "$SUCCESS_URL" ] && [[ "$SUCCESS_URL" != */ ]]; then
- SUCCESS_URL=${SUCCESS_URL}/
- fi
-
- if [ -n "$FAILURE_URL" ] && [[ "$FAILURE_URL" != */ ]]; then
- FAILURE_URL=${FAILURE_URL}/
- fi
-
- host_name=$(hostname -f | tr '[:upper:]' '[:lower:]')
- msg=$host_name
-
- if [ -n "$1" ]; then
- echo "[ERROR] $1"
- echo "[ERROR]-----------------------------------------------------"
- echo "[ERROR] Test node bootstrap failed"
- echo "[ERROR]-----------------------------------------------------"
- msg=$1
-
- if [ -z "$FAILURE_URL" ]; then
- exit 1
- fi
-
- reportFolder=${FAILURE_URL}${host_name}
- reportFile=$reportFolder/__error__
- else
- echo "[INFO]-----------------------------------------------------"
- echo "[INFO] Test node bootstrap successfully completed"
- echo "[INFO]-----------------------------------------------------"
-
- if [ -z "$SUCCESS_URL" ]; then
- exit 0
- fi
-
- reportFolder=${SUCCESS_URL}${host_name}
- reportFile=$reportFolder/__success__
- fi
-
- echo $msg > /opt/bootstrap-result
-
- aws s3 rm --recursive $reportFolder
- if [ $? -ne 0 ]; then
- echo "[ERROR] Failed to drop report folder: $reportFolder"
- fi
-
- aws s3 cp --sse AES256 /opt/bootstrap-result $reportFile
- if [ $? -ne 0 ]; then
- echo "[ERROR] Failed to report bootstrap result to: $reportFile"
- fi
-
- rm -f /opt/bootstrap-result
-
- if [ -n "$1" ]; then
- exit 1
- fi
-
- exit 0
-}
-
-# Downloads specified package
-downloadPackage()
-{
- echo "[INFO] Downloading $3 package from $1 into $2"
-
- for i in 0 9;
- do
- if [[ "$1" == s3* ]]; then
- aws s3 cp $1 $2
- code=$?
- else
- curl "$1" -o "$2"
- code=$?
- fi
-
- if [ $code -eq 0 ]; then
- echo "[INFO] $3 package successfully downloaded from $1 into $2"
- return 0
- fi
-
- echo "[WARN] Failed to download $3 package from $i attempt, sleeping extra 5sec"
- sleep 5s
- done
-
- terminate "All 10 attempts to download $3 package from $1 are failed"
-}
-
-# Downloads and setup JDK
-setupJava()
-{
- rm -Rf /opt/java /opt/jdk.tar.gz
-
- echo "[INFO] Downloading 'jdk'"
- wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "$JDK_DOWNLOAD_URL" -O /opt/jdk.tar.gz
- if [ $? -ne 0 ]; then
- terminate "Failed to download 'jdk'"
- fi
-
- echo "[INFO] Untaring 'jdk'"
- tar -xvzf /opt/jdk.tar.gz -C /opt
- if [ $? -ne 0 ]; then
- terminate "Failed to untar 'jdk'"
- fi
-
- rm -Rf /opt/jdk.tar.gz
-
- unzipDir=$(ls /opt | grep "jdk")
- if [ "$unzipDir" != "java" ]; then
- mv /opt/$unzipDir /opt/java
- fi
-}
-
-# Downloads and setup AWS CLI
-setupAWSCLI()
-{
- echo "[INFO] Installing 'awscli'"
- pip install --upgrade awscli
- if [ $? -eq 0 ]; then
- return 0
- fi
-
- echo "[ERROR] Failed to install 'awscli' using pip"
- echo "[INFO] Trying to install awscli using zip archive"
- echo "[INFO] Downloading awscli zip"
-
- downloadPackage "$AWS_CLI_DOWNLOAD_URL" "/opt/awscli-bundle.zip" "awscli"
-
- echo "[INFO] Unzipping awscli zip"
- unzip /opt/awscli-bundle.zip -d /opt
- if [ $? -ne 0 ]; then
- terminate "Failed to unzip awscli zip"
- fi
-
- rm -Rf /opt/awscli-bundle.zip
-
- echo "[INFO] Installing awscli"
- /opt/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws
- if [ $? -ne 0 ]; then
- terminate "Failed to install awscli"
- fi
-
- echo "[INFO] Successfully installed awscli from zip archive"
-}
-
-# Setup all the pre-requisites (packages, settings and etc.)
-setupPreRequisites()
-{
- echo "[INFO] Installing 'wget' package"
- yum -y install wget
- if [ $? -ne 0 ]; then
- terminate "Failed to install 'wget' package"
- fi
-
- echo "[INFO] Installing 'net-tools' package"
- yum -y install net-tools
- if [ $? -ne 0 ]; then
- terminate "Failed to install 'net-tools' package"
- fi
-
- echo "[INFO] Installing 'python' package"
- yum -y install python
- if [ $? -ne 0 ]; then
- terminate "Failed to install 'python' package"
- fi
-
- echo "[INFO] Installing 'unzip' package"
- yum -y install unzip
- if [ $? -ne 0 ]; then
- terminate "Failed to install 'unzip' package"
- fi
-
- downloadPackage "https://bootstrap.pypa.io/get-pip.py" "/opt/get-pip.py" "get-pip.py"
-
- echo "[INFO] Installing 'pip'"
- python /opt/get-pip.py
- if [ $? -ne 0 ]; then
- terminate "Failed to install 'pip'"
- fi
-}
-
-# Downloads and setup tests package
-setupTestsPackage()
-{
- downloadPackage "$TESTS_PACKAGE_DONLOAD_URL" "/opt/ignite-cassandra-tests.zip" "Tests"
-
- rm -Rf /opt/ignite-cassandra-tests
-
- unzip /opt/ignite-cassandra-tests.zip -d /opt
- if [ $? -ne 0 ]; then
- terminate "Failed to unzip tests package"
- fi
-
- rm -f /opt/ignite-cassandra-tests.zip
-
- unzipDir=$(ls /opt | grep "ignite-cassandra")
- if [ "$unzipDir" != "ignite-cassandra-tests" ]; then
- mv /opt/$unzipDir /opt/ignite-cassandra-tests
- fi
-
- find /opt/ignite-cassandra-tests -type f -name "*.sh" -exec chmod ug+x {} \;
-
- . /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "test"
-
- setupNTP
-
- echo "[INFO] Starting logs collector daemon"
-
- HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]')
- /opt/ignite-cassandra-tests/bootstrap/aws/logs-collector.sh "$S3_LOGS_TRIGGER" "$S3_TESTS_LOGS/$HOST_NAME" "/opt/ignite-cassandra-tests/logs" > /opt/logs-collector.log &
-
- echo "[INFO] Logs collector daemon started: $!"
-
- echo "----------------------------------------------------------------------------------------"
- printInstanceInfo
- echo "----------------------------------------------------------------------------------------"
- tagInstance
- bootstrapGangliaAgent "test" 8643
-
- ###################################################
- # Extra configuration specific only for test node #
- ###################################################
-
- echo "[INFO] Installing bc package"
-
- yum -y install bc
-
- if [ $? -ne 0 ]; then
- terminate "Failed to install bc package"
- fi
-
- echo "[INFO] Installing zip package"
-
- yum -y install zip
-
- if [ $? -ne 0 ]; then
- terminate "Failed to install zip package"
- fi
-
- echo "[INFO] Creating 'ignite' group"
- exists=$(cat /etc/group | grep ignite)
- if [ -z "$exists" ]; then
- groupadd ignite
- if [ $? -ne 0 ]; then
- terminate "Failed to create 'ignite' group"
- fi
- fi
-
- echo "[INFO] Creating 'ignite' user"
- exists=$(cat /etc/passwd | grep ignite)
- if [ -z "$exists" ]; then
- useradd -g ignite ignite
- if [ $? -ne 0 ]; then
- terminate "Failed to create 'ignite' user"
- fi
- fi
-
- mkdir -p /opt/ignite-cassandra-tests/logs
- chown -R ignite:ignite /opt/ignite-cassandra-tests
-
- echo "export JAVA_HOME=/opt/java" >> $1
- echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> $1
-}
-
-###################################################################################################################
-
-echo "[INFO]-----------------------------------------------------------------"
-echo "[INFO] Bootstrapping Tests node"
-echo "[INFO]-----------------------------------------------------------------"
-
-setupPreRequisites
-setupJava
-setupAWSCLI
-setupTestsPackage "/root/.bash_profile"
-
-cmd="/opt/ignite-cassandra-tests/bootstrap/aws/tests/tests-manager.sh"
-
-#sudo -u ignite -g ignite sh -c "$cmd > /opt/ignite-cassandra-tests/tests-manager" &
-
-$cmd > /opt/ignite-cassandra-tests/logs/tests-manager.log &
-
-terminate
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/tests/tests-manager.sh
----------------------------------------------------------------------
diff --git a/modules/cassandra/src/test/bootstrap/aws/tests/tests-manager.sh b/modules/cassandra/src/test/bootstrap/aws/tests/tests-manager.sh
deleted file mode 100644
index c0f5d6b..0000000
--- a/modules/cassandra/src/test/bootstrap/aws/tests/tests-manager.sh
+++ /dev/null
@@ -1,458 +0,0 @@
-#!/bin/sh
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# -----------------------------------------------------------------------------------------------
-# Tests manager daemon
-# -----------------------------------------------------------------------------------------------
-# Script is launched in background by all nodes of Tests cluster and
-# periodically (each 30 seconds) checks if specific S3 trigger file was created or
-# its timestamp was changed. Such an event serve as a trigger for the script to start
-# preparing to run load tests.
-# -----------------------------------------------------------------------------------------------
-
-#profile=/home/ignite/.bash_profile
-profile=/root/.bash_profile
-
-. $profile
-. /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "test"
-
-# Switch test node to IDLE state
-switchToIdleState()
-{
- if [ "$NODE_STATE" != "IDLE" ]; then
- echo "[INFO] Switching node to IDLE state"
- dropStateFlag "$S3_TESTS_WAITING" "$S3_TESTS_PREPARING" "$S3_TESTS_RUNNING"
- createStateFlag "$S3_TESTS_IDLE"
- NODE_STATE="IDLE"
- echo "[INFO] Node was switched to IDLE state"
- fi
-}
-
-# Switch test node to PREPARING state
-switchToPreparingState()
-{
- if [ "$NODE_STATE" != "PREPARING" ]; then
- echo "[INFO] Switching node to PREPARING state"
- dropStateFlag "$S3_TESTS_WAITING" "$S3_TESTS_IDLE" "$S3_TESTS_RUNNING"
- createStateFlag "$S3_TESTS_PREPARING"
- NODE_STATE="PREPARING"
- echo "[INFO] Node was switched to PREPARING state"
- fi
-}
-
-# Switch test node to WAITING state
-switchToWaitingState()
-{
- if [ "$NODE_STATE" != "WAITING" ]; then
- echo "[INFO] Switching node to WAITING state"
- dropStateFlag "$S3_TESTS_IDLE" "$S3_TESTS_PREPARING" "$S3_TESTS_RUNNING"
- createStateFlag "$S3_TESTS_WAITING"
- NODE_STATE="WAITING"
- echo "[INFO] Node was switched to WAITING state"
- fi
-}
-
-# Switch test node to RUNNING state
-switchToRunningState()
-{
- if [ "$NODE_STATE" != "RUNNING" ]; then
- echo "[INFO] Switching node to RUNNING state"
- dropStateFlag "$S3_TESTS_IDLE" "$S3_TESTS_PREPARING" "$S3_TESTS_WAITING"
- createStateFlag "$S3_TESTS_RUNNING"
- NODE_STATE="RUNNING"
- echo "[INFO] Node was switched to RUNNING state"
- fi
-}
-
-# Creates appropriate state flag for the node in S3
-createStateFlag()
-{
- HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]')
-
- aws s3 cp --sse AES256 /etc/hosts ${1}${HOST_NAME}
- if [ $? -ne 0 ]; then
- terminate "Failed to create state flag: ${1}${HOST_NAME}"
- fi
-}
-
-# Drops appropriate state flag for the node in S3
-dropStateFlag()
-{
- HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]')
-
- for flagUrl in "$@"
- do
- exists=$(aws s3 ls ${flagUrl}${HOST_NAME})
- if [ -n "$exists" ]; then
- aws s3 rm ${flagUrl}${HOST_NAME}
- if [ $? -ne 0 ]; then
- terminate "Failed to drop state flag: ${flagUrl}${HOST_NAME}"
- fi
- fi
- done
-}
-
-# Removes tests summary report from S3
-dropTestsSummary()
-{
- exists=$(aws s3 ls $S3_TESTS_SUMMARY)
- if [ -z "$exists" ]; then
- return 0
- fi
-
- aws s3 rm $S3_TESTS_SUMMARY
- if [ $? -ne 0 ]; then
- terminate "Failed to drop tests summary info: $S3_TESTS_SUMMARY"
- fi
-}
-
-# Recreate all the necessary Cassandra artifacts before running Load tests
-recreateCassandraArtifacts()
-{
- /opt/ignite-cassandra-tests/recreate-cassandra-artifacts.sh
- if [ $? -ne 0 ]; then
- terminate "Failed to recreate Cassandra artifacts"
- fi
-}
-
-# Setups Cassandra seeds for this Tests node being able to connect to Cassandra.
-# Looks for the information in S3 about already up and running Cassandra cluster nodes.
-setupCassandraSeeds()
-{
- if [ $CASSANDRA_NODES_COUNT -eq 0 ]; then
- return 0
- fi
-
- setupClusterSeeds "cassandra"
-
- CASSANDRA_SEEDS1=$(echo $CLUSTER_SEEDS | sed -r "s/ /,/g")
- CASSANDRA_SEEDS2=
-
- CLUSTER_SEEDS=($CLUSTER_SEEDS)
- count=${#CLUSTER_SEEDS[@]}
-
- for (( i=0; i<=$(( $count -1 )); i++ ))
- do
- seed=${CLUSTER_SEEDS[$i]}
- CASSANDRA_SEEDS2="${CASSANDRA_SEEDS2}<value>$seed<\/value>"
- done
-
- echo "[INFO] Using Cassandra seeds: $CASSANDRA_SEEDS1"
-
- echo "contact.points=$CASSANDRA_SEEDS1" > /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/connection.properties
-
- cat /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template.xml | sed -r "s/\\\$\{CASSANDRA_SEEDS\}/$CASSANDRA_SEEDS2/g" > /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template1.xml
-}
-
-# Setups Ignite nodes for this Tests node being able to connect to Ignite.
-# Looks for the information in S3 about already up and running Cassandra cluster nodes.
-setupIgniteSeeds()
-{
- if [ $IGNITE_NODES_COUNT -eq 0 ]; then
- return 0
- fi
-
- setupClusterSeeds "ignite"
-
- CLUSTER_SEEDS=($CLUSTER_SEEDS)
- count=${#CLUSTER_SEEDS[@]}
-
- IGNITE_SEEDS=
-
- for (( i=0; i<=$(( $count -1 )); i++ ))
- do
- seed=${CLUSTER_SEEDS[$i]}
- IGNITE_SEEDS="${IGNITE_SEEDS}<value>$seed<\/value>"
- done
-
- echo "[INFO] Using Ignite seeds: $IGNITE_SEEDS"
-
- cat /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template1.xml | sed -r "s/\\\$\{IGNITE_SEEDS\}/$IGNITE_SEEDS/g" > /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/persistence/primitive/ignite-remote-client-config.xml
- rm -f /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template1.xml
-}
-
-# Setups Cassandra credentials to connect to Cassandra cluster
-setupCassandraCredentials()
-{
- echo "admin.user=cassandra" > /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties
- echo "admin.password=cassandra" >> /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties
- echo "regular.user=cassandra" >> /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties
- echo "regular.password=cassandra" >> /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties
-}
-
-# Triggering first time tests execution for all nodes in the Tests cluster
-triggerFirstTimeTestsExecution()
-{
- if [ -z "$TESTS_TYPE" ]; then
- return 0
- fi
-
- tryToGetFirstNodeLock
- if [ $? -ne 0 ]; then
- return 0
- fi
-
- sleep 30s
-
- echo "[INFO] Triggering first time tests execution"
-
- echo "TESTS_TYPE=$TESTS_TYPE" > /opt/ignite-cassandra-tests/tests-trigger
- echo "#--------------------------------------------------" >> /opt/ignite-cassandra-tests/tests-trigger
- echo "" >> /opt/ignite-cassandra-tests/tests-trigger
- cat /opt/ignite-cassandra-tests/settings/tests.properties >> /opt/ignite-cassandra-tests/tests-trigger
-
- aws s3 cp --sse AES256 /opt/ignite-cassandra-tests/tests-trigger $S3_TESTS_TRIGGER
- code=$?
-
- rm -f /opt/ignite-cassandra-tests/tests-trigger
-
- if [ $code -ne 0 ]; then
- terminate "Failed to create tests trigger: $S3_TESTS_TRIGGER"
- fi
-}
-
-# Cleans previously created logs from S3
-cleanPreviousLogs()
-{
- for logFile in /opt/ignite-cassandra-tests/logs/*
- do
- managerLog=$(echo $logFile | grep "tests-manager")
- if [ -z "$managerLog" ]; then
- rm -Rf $logFile
- fi
- done
-
- HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]')
-
- aws s3 rm --recursive ${S3_TESTS_FAILURE}${HOST_NAME}
- aws s3 rm --recursive ${S3_TESTS_SUCCESS}${HOST_NAME}
-}
-
-# Uploads tests logs to S3
-uploadTestsLogs()
-{
- HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]')
-
- if [ -f "/opt/ignite-cassandra-tests/logs/__success__" ]; then
- logsFolder=${S3_TESTS_SUCCESS}${HOST_NAME}
- else
- logsFolder=${S3_TESTS_FAILURE}${HOST_NAME}
- fi
-
- aws s3 rm --recursive $logsFolder
- if [ $? -ne 0 ]; then
- echo "[ERROR] Failed to drop logs folder: $logsFolder"
- fi
-
- if [ -d "/opt/ignite-cassandra-tests/logs" ]; then
- aws s3 sync --sse AES256 /opt/ignite-cassandra-tests/logs $logsFolder
- if [ $? -ne 0 ]; then
- echo "[ERROR] Failed to export tests logs to: $logsFolder"
- fi
- fi
-}
-
-# Runs tests-report.sh to prepare tests summary report
-buildTestsSummaryReport()
-{
- reportScript=$(readlink -m $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/tests-report.sh)
- $reportScript
-
- if [ -n "$S3_LOGS_TRIGGER" ]; then
- aws s3 cp --sse AES256 /etc/hosts $S3_LOGS_TRIGGER
- if [ $? -ne 0 ]; then
- echo "[ERROR] Failed to trigger logs collection"
- fi
- fi
-}
-
-# Running load tests
-runLoadTests()
-{
- cd /opt/ignite-cassandra-tests
-
- if [ "$TESTS_TYPE" == "ignite" ]; then
- echo "[INFO] Running Ignite load tests"
- ./ignite-load-tests.sh &
- else
- echo "[INFO] Running Cassandra load tests"
- ./cassandra-load-tests.sh &
- fi
-
- testsJobId=$!
-
- echo "[INFO] Tests job id: $testsJobId"
-
- sleep 1m
-
- LOGS_SNAPSHOT=$(ls -al /opt/ignite-cassandra-tests/logs)
- LOGS_SNAPSHOT_TIME=$(date +%s)
-
- TERMINATED=
-
- # tests monitoring
- while true; do
- proc=$(ps -ef | grep java | grep "org.apache.ignite.tests")
- if [ -z "$proc" ]; then
- break
- fi
-
- NEW_LOGS_SNAPSHOT=$(ls -al /opt/ignite-cassandra-tests/logs)
- NEW_LOGS_SNAPSHOT_TIME=$(date +%s)
-
- # if logs state updated it means that tests are running and not stuck
- if [ "$LOGS_SNAPSHOT" != "$NEW_LOGS_SNAPSHOT" ]; then
- LOGS_SNAPSHOT=$NEW_LOGS_SNAPSHOT
- LOGS_SNAPSHOT_TIME=$NEW_LOGS_SNAPSHOT_TIME
- continue
- fi
-
- duration=$(( $NEW_LOGS_SNAPSHOT_TIME-$LOGS_SNAPSHOT_TIME ))
- duration=$(( $duration/60 ))
-
- # if logs wasn't updated during 5min it means that load tests stuck
- if [ $duration -gt 5 ]; then
- proc=($proc)
- kill -9 ${proc[1]}
- TERMINATED="true"
- break
- fi
-
- echo "[INFO] Waiting extra 30sec for load tests to complete"
-
- sleep 30s
- done
-
- rm -f /opt/ignite-cassandra-tests/logs/tests.properties
- cp /opt/ignite-cassandra-tests/settings/tests.properties /opt/ignite-cassandra-tests/logs
-
- if [ "$TERMINATED" == "true" ]; then
- echo "[ERROR] Load tests stuck, tests process terminated"
- echo "Load tests stuck, tests process terminated" > /opt/ignite-cassandra-tests/logs/__error__
- return 0
- fi
-
- failed=
- if [ "$TESTS_TYPE" == "cassandra" ]; then
- failed=$(cat /opt/ignite-cassandra-tests/cassandra-load-tests.log | grep "load tests execution failed")
- else
- failed=$(cat /opt/ignite-cassandra-tests/ignite-load-tests.log | grep "load tests execution failed")
- fi
-
- if [ -n "$failed" ]; then
- echo "[ERROR] Load tests execution failed"
- echo "Load tests execution failed" > /opt/ignite-cassandra-tests/logs/__error__
- else
- echo "[INFO] Load tests execution successfully completed"
- echo "Load tests execution successfully completed" > /opt/ignite-cassandra-tests/logs/__success__
- fi
-}
-
-#######################################################################################################
-
-sleep 1m
-
-NODE_STATE=
-TRIGGER_STATE=
-
-printInstanceInfo
-setupCassandraCredentials
-switchToIdleState
-
-triggerFirstTimeTestsExecution
-
-registerNode
-
-while true; do
- # switching state to IDLE
- switchToIdleState
-
- sleep 30s
-
- NEW_TRIGGER_STATE=$(aws s3 ls $S3_TESTS_TRIGGER | xargs)
- if [ -z "$NEW_TRIGGER_STATE" ] || [ "$NEW_TRIGGER_STATE" == "$TRIGGER_STATE" ]; then
- continue
- fi
-
- echo "----------------------------------------------------------------------"
- echo "[INFO] Tests trigger changed"
- echo "----------------------------------------------------------------------"
- echo "[INFO] Old trigger: $TRIGGER_STATE"
- echo "----------------------------------------------------------------------"
- echo "[INFO] New trigger: $NEW_TRIGGER_STATE"
- echo "----------------------------------------------------------------------"
-
- TRIGGER_STATE=$NEW_TRIGGER_STATE
-
- aws s3 cp $S3_TESTS_TRIGGER /opt/ignite-cassandra-tests/tests-trigger
- if [ $? -ne 0 ]; then
- echo "[ERROR] Failed to download tests trigger info from: $S3_TESTS_TRIGGER"
- continue
- fi
-
- TESTS_TYPE=$(cat /opt/ignite-cassandra-tests/tests-trigger | grep TESTS_TYPE | xargs | sed -r "s/TESTS_TYPE=//g")
- if [ "$TESTS_TYPE" != "ignite" ] && [ "$TESTS_TYPE" != "cassandra" ]; then
- rm -f /opt/ignite-cassandra-tests/tests-trigger
- echo "[ERROR] Incorrect tests type specified in the trigger info: $S3_TESTS_TRIGGER"
- continue
- fi
-
- rm -f /opt/ignite-cassandra-tests/settings/tests.properties
- mv -f /opt/ignite-cassandra-tests/tests-trigger /opt/ignite-cassandra-tests/settings/tests.properties
-
- waitAllTestNodesCompletedTests
-
- # switching state to PREPARING
- switchToPreparingState
-
- waitAllClusterNodesReady "cassandra"
- waitAllClusterNodesReady "ignite"
- setupCassandraSeeds
- setupIgniteSeeds
-
- cleanPreviousLogs
-
- tryToGetFirstNodeLock
- if [ $? -eq 0 ]; then
- dropTestsSummary
- recreateCassandraArtifacts
- fi
-
- # switching state to WAITING
- switchToWaitingState
-
- waitAllClusterNodesReady "test"
-
- if [ "$FIRST_NODE_LOCK" == "true" ]; then
- aws s3 rm $S3_TESTS_TRIGGER
- fi
-
- # switching state to RUNNING
- switchToRunningState
-
- runLoadTests
- uploadTestsLogs
-
- tryToGetFirstNodeLock
- if [ $? -eq 0 ]; then
- waitAllTestNodesCompletedTests
- buildTestsSummaryReport
- removeFirstNodeLock
- fi
-done
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/bootstrap/aws/tests/tests-report.sh
----------------------------------------------------------------------
diff --git a/modules/cassandra/src/test/bootstrap/aws/tests/tests-report.sh b/modules/cassandra/src/test/bootstrap/aws/tests/tests-report.sh
deleted file mode 100644
index 1576d57..0000000
--- a/modules/cassandra/src/test/bootstrap/aws/tests/tests-report.sh
+++ /dev/null
@@ -1,499 +0,0 @@
-#!/bin/sh
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# -----------------------------------------------------------------------------------------------
-# Tests report builder
-# -----------------------------------------------------------------------------------------------
-# Script is used to analyze load tests logs collected from all 'Tests' cluster nodes and build
-# summary report
-# -----------------------------------------------------------------------------------------------
-
-#profile=/home/ignite/.bash_profile
-profile=/root/.bash_profile
-
-. $profile
-. /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "test"
-
-# Building tests summary report
-reportTestsSummary()
-{
- echo "[INFO] Preparing tests results summary"
-
- TESTS_SUMMARY_DIR=/opt/ignite-cassandra-tests/tests-summary
- SUCCEED_NODES_FILE=$TESTS_SUMMARY_DIR/succeed-nodes
- SUCCEED_NODES_DIR=$TESTS_SUMMARY_DIR/succeed
- FAILED_NODES_FILE=$TESTS_SUMMARY_DIR/failed-nodes
- FAILED_NODES_DIR=$TESTS_SUMMARY_DIR/failed
- REPORT_FILE=$TESTS_SUMMARY_DIR/report.txt
-
- rm -Rf $TESTS_SUMMARY_DIR
- mkdir -p $TESTS_SUMMARY_DIR
- mkdir -p $SUCCEED_NODES_DIR
- mkdir -p $FAILED_NODES_DIR
-
- aws s3 ls $S3_TESTS_SUCCESS | sed -r "s/PRE //g" | sed -r "s/ //g" | sed -r "s/\///g" > $SUCCEED_NODES_FILE
- aws s3 ls $S3_TESTS_FAILURE | sed -r "s/PRE //g" | sed -r "s/ //g" | sed -r "s/\///g" > $FAILED_NODES_FILE
-
- succeedCount=$(cat $SUCCEED_NODES_FILE | wc -l)
- failedCount=$(cat $FAILED_NODES_FILE | wc -l)
- count=$(( $succeedCount+$failedCount ))
-
- echo "Test type : $TESTS_TYPE" > $REPORT_FILE
- echo "Test nodes count : $count" >> $REPORT_FILE
- echo "Test nodes succeed: $succeedCount" >> $REPORT_FILE
- echo "Test nodes failed : $failedCount" >> $REPORT_FILE
- echo "----------------------------------------------------------------------------------------------" >> $REPORT_FILE
-
- if [ $succeedCount -gt 0 ]; then
- echo "Succeed test nodes |" >> $REPORT_FILE
- echo "-------------------" >> $REPORT_FILE
- cat $SUCCEED_NODES_FILE >> $REPORT_FILE
- echo "----------------------------------------------------------------------------------------------" >> $REPORT_FILE
-
- aws s3 sync --delete $S3_TESTS_SUCCESS $SUCCEED_NODES_DIR
- if [ $? -ne 0 ]; then
- echo "[ERROR] Failed to get succeed tests details"
- else
- reportSucceedTestsStatistics "$REPORT_FILE" "$SUCCEED_NODES_DIR"
- fi
- fi
-
- if [ $failedCount -gt 0 ]; then
- echo "Failed test nodes |" >> $REPORT_FILE
- echo "------------------" >> $REPORT_FILE
- cat $FAILED_NODES_FILE >> $REPORT_FILE
- echo "----------------------------------------------------------------------------------------------" >> $REPORT_FILE
-
- aws sync --delete $S3_TESTS_FAILURE $FAILED_NODES_DIR
- if [ $? -ne 0 ]; then
- echo "[ERROR] Failed to get failed tests details"
- else
- reportFailedTestsDetailes "$REPORT_FILE" "$FAILED_NODES_DIR"
- fi
- fi
-
- rm -f $HOME/tests-summary.zip
-
- pushd $TESTS_SUMMARY_DIR
-
- zip -r -9 $HOME/tests-summary.zip .
- code=$?
-
- rm -Rf $TESTS_SUMMARY_DIR
-
- popd
-
- if [ $code -ne 0 ]; then
- echo "-------------------------------------------------------------------------------------"
- echo "[ERROR] Failed to create tests summary zip archive $HOME/tests-summary.zip for $TESTS_SUMMARY_DIR"
- echo "-------------------------------------------------------------------------------------"
- return 1
- fi
-
- aws s3 cp --sse AES256 $HOME/tests-summary.zip $S3_TESTS_SUMMARY
- if [ $? -ne 0 ]; then
- echo "-------------------------------------------------------------------------------------"
- echo "[ERROR] Failed to uploat tests summary archive to: $S3_TESTS_SUMMARY"
- echo "-------------------------------------------------------------------------------------"
- else
- echo "-------------------------------------------------------------------------------------"
- echo "[INFO] Tests results summary uploaded to: $S3_TESTS_SUMMARY"
- echo "-------------------------------------------------------------------------------------"
- fi
-
- rm -f $HOME/tests-summary.zip
-}
-
-# Creates report for succeed tests
-reportSucceedTestsStatistics()
-{
- writeMsg="0"
- writeErrors="0"
- writeSpeed="0"
- blkWriteMsg="0"
- blkWriteErrors="0"
- blkWriteSpeed="0"
- readMsg="0"
- readErrors="0"
- readSpeed="0"
- blkReadMsg="0"
- blkReadErrors="0"
- blkReadSpeed="0"
-
- writeErrNodes=
- blkWriteErrNodes=
- readErrNodes=
- blkReadErrNodes=
-
- tmpFile=`mktemp`
-
- for dir in $2/*
- do
- node=$(echo $dir | sed -r "s/^.*\///g")
- echo "-------------------------------------------------------------------------------------"
- echo "[INFO] Gathering statistics from $node test node"
- echo "-------------------------------------------------------------------------------------"
-
- logFile=$(ls $dir | grep "${TESTS_TYPE}-load-tests.log" | head -1)
- if [ -z "$logFile" ]; then
- echo "[WARN] Node $node marked as succeeded, but it doesn't have \"${TESTS_TYPE}-load-tests.log\" tests results summary file"
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Node $node marked as succeeded," >> $tmpFile
- echo "but it doesn't have \"${TESTS_TYPE}-load-tests.log\" tests results summary file" >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- continue
- fi
-
- logFile=$dir/$logFile
- if [ ! -f "$logFile" ]; then
- echo "[WARN] Node $node marked as succeeded, but it doesn't have \"${TESTS_TYPE}-load-tests.log\" tests results summary file"
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Node $node marked as succeeded," >> $tmpFile
- echo "but it doesn't have \"${TESTS_TYPE}-load-tests.log\" tests results summary file" >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- continue
- fi
-
- cnt=$(cat $logFile | grep "^WRITE messages" | sed -r "s/WRITE messages: //g" | xargs)
- if [ -n "$cnt" ]; then
- writeMsg=$(bc <<< "$writeMsg + $cnt")
- if [ $cnt -ne 0 ]; then
- echo "[INFO] WRITE messages: $cnt"
- else
- echo "[WARN] WRITE messages count is zero for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "WRITE messages count is zero for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
- else
- echo "[WARN] Failed to detect WRITE messages count for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Failed to detect WRITE messages count for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
-
- cnt=$(cat $logFile | grep "^WRITE errors" | sed -r "s/WRITE errors: //g" | sed -r "s/,.*//g" | xargs)
- if [ -n "$cnt" ]; then
- echo "[INFO] WRITE errors: $cnt"
- writeErrors=$(bc <<< "$writeErrors + $cnt")
- if [ $cnt -ne 0 ]; then
- if [ -n "$writeErrNodes" ]; then
- writeErrNodes="${writeErrNodes}, "
- fi
- writeErrNodes="${writeErrNodes}${node}"
- fi
- else
- echo "[WARN] Failed to detect WRITE errors count for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Failed to detect WRITE errors count for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
-
- cnt=$(cat $logFile | grep "^WRITE speed" | sed -r "s/WRITE speed: //g" | sed -r "s/ msg\/sec//g" | xargs)
- if [ -n "$cnt" ]; then
- writeSpeed=$(bc <<< "$writeSpeed + $cnt")
- if [ $cnt -ne 0 ]; then
- echo "[INFO] WRITE speed: $cnt msg/sec"
- else
- echo "[WARN] WRITE speed is zero for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "WRITE speed is zero for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
- else
- echo "[WARN] Failed to detect WRITE speed for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Failed to detect WRITE speed for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
-
- cnt=$(cat $logFile | grep "^BULK_WRITE messages" | sed -r "s/BULK_WRITE messages: //g" | xargs)
- if [ -n "$cnt" ]; then
- blkWriteMsg=$(bc <<< "$blkWriteMsg + $cnt")
- if [ $cnt -ne 0 ]; then
- echo "[INFO] BULK_WRITE messages: $cnt"
- else
- echo "[WARN] BULK_WRITE messages count is zero for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "BULK_WRITE messages count is zero for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
- else
- echo "[WARN] Failed to detect BULK_WRITE messages count for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Failed to detect BULK_WRITE messages count for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
-
- cnt=$(cat $logFile | grep "^BULK_WRITE errors" | sed -r "s/BULK_WRITE errors: //g" | sed -r "s/,.*//g" | xargs)
- if [ -n "$cnt" ]; then
- blkWriteErrors=$(bc <<< "$blkWriteErrors + $cnt")
- echo "[INFO] BULK_WRITE errors: $cnt"
- if [ $cnt -ne 0 ]; then
- if [ -n "$blkWriteErrNodes" ]; then
- blkWriteErrNodes="${blkWriteErrNodes}, "
- fi
- blkWriteErrNodes="${blkWriteErrNodes}${node}"
- fi
- else
- echo "[WARN] Failed to detect BULK_WRITE errors count for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Failed to detect BULK_WRITE errors count for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
-
- cnt=$(cat $logFile | grep "^BULK_WRITE speed" | sed -r "s/BULK_WRITE speed: //g" | sed -r "s/ msg\/sec//g" | xargs)
- if [ -n "$cnt" ]; then
- blkWriteSpeed=$(bc <<< "$blkWriteSpeed + $cnt")
- if [ $cnt -ne 0 ]; then
- echo "[INFO] BULK_WRITE speed: $cnt msg/sec"
- else
- echo "[WARN] BULK_WRITE speed is zero for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "BULK_WRITE speed is zero for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
- else
- echo "[WARN] Failed to detect BULK_WRITE speed for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Failed to detect BULK_WRITE speed for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
-
- cnt=$(cat $logFile | grep "^READ messages" | sed -r "s/READ messages: //g" | xargs)
- if [ -n "$cnt" ]; then
- readMsg=$(bc <<< "$readMsg + $cnt")
- if [ $cnt -ne 0 ]; then
- echo "[INFO] READ messages: $cnt"
- else
- echo "[WARN] READ messages count is zero for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "READ messages count is zero for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
- else
- echo "[WARN] Failed to detect READ messages count for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Failed to detect READ messages count for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
-
- cnt=$(cat $logFile | grep "^READ errors" | sed -r "s/READ errors: //g" | sed -r "s/,.*//g" | xargs)
- if [ -n "$cnt" ]; then
- readErrors=$(bc <<< "$readErrors + $cnt")
- echo "[INFO] READ errors: $cnt"
- if [ $cnt -ne 0 ]; then
- if [ -n "$readErrNodes" ]; then
- blkWriteErrNodes="${readErrNodes}, "
- fi
- readErrNodes="${readErrNodes}${node}"
- fi
- else
- echo "[WARN] Failed to detect READ errors count for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Failed to detect READ errors count for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
-
- cnt=$(cat $logFile | grep "^READ speed" | sed -r "s/READ speed: //g" | sed -r "s/ msg\/sec//g" | xargs)
- if [ -n "$cnt" ]; then
- readSpeed=$(bc <<< "$readSpeed + $cnt")
- if [ $cnt -ne 0 ]; then
- echo "[INFO] READ speed: $cnt msg/sec"
- else
- echo "[WARN] READ speed is zero for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "READ speed is zero for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
- else
- echo "[WARN] Failed to detect READ speed for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Failed to detect READ speed for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
-
- cnt=$(cat $logFile | grep "^BULK_READ messages" | sed -r "s/BULK_READ messages: //g" | xargs)
- if [ -n "$cnt" ]; then
- blkReadMsg=$(bc <<< "$blkReadMsg + $cnt")
- if [ $cnt -ne 0 ]; then
- echo "[INFO] BULK_READ messages: $cnt"
- else
- echo "[WARN] BULK_READ messages count is zero for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "BULK_READ messages count is zero for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
- else
- echo "[WARN] Failed to detect BULK_READ messages count for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Failed to detect BULK_READ messages count for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
-
- cnt=$(cat $logFile | grep "^BULK_READ errors" | sed -r "s/BULK_READ errors: //g" | sed -r "s/,.*//g" | xargs)
- if [ -n "$cnt" ]; then
- blkReadErrors=$(bc <<< "$blkReadErrors + $cnt")
- echo "[INFO] BULK_READ errors: $cnt"
- if [ $cnt -ne 0 ]; then
- if [ -n "$blkReadErrNodes" ]; then
- blkReadErrNodes="${blkReadErrNodes}, "
- fi
- blkReadErrNodes="${blkReadErrNodes}${node}"
- fi
- else
- echo "[WARN] Failed to detect BULK_READ errors count for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Failed to detect BULK_READ errors count for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
-
- cnt=$(cat $logFile | grep "^BULK_READ speed" | sed -r "s/BULK_READ speed: //g" | sed -r "s/ msg\/sec//g" | xargs)
- if [ -n "$cnt" ]; then
- blkReadSpeed=$(bc <<< "$blkReadSpeed + $cnt")
- if [ $cnt -ne 0 ]; then
- echo "[INFO] BULK_READ speed: $cnt msg/sec"
- else
- echo "[WARN] BULK_READ speed is zero for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "BULK_READ speed is zero for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
- else
- echo "[WARN] Failed to detect BULK_READ speed for $node node. This test probably failed."
- echo "WARNING |" >> $tmpFile
- echo "--------" >> $tmpFile
- echo "Failed to detect BULK_READ speed for $node node. This test probably failed." >> $tmpFile
- echo "----------------------------------------------------------------------------------------------" >> $tmpFile
- fi
- done
-
- echo "-------------------------------------------------------------------------------------"
-
- echo "WRITE test metrics |" >> $1
- echo "-------------------" >> $1
- echo "Messages: $writeMsg" >> $1
- echo "Speed : $writeSpeed msg/sec" >> $1
- echo "Errors : $writeErrors" >> $1
- echo "----------------------------------------------------------------------------------------------" >> $1
-
- echo "BULK_WRITE test metrics |" >> $1
- echo "------------------------" >> $1
- echo "Messages: $blkWriteMsg" >> $1
- echo "Speed : $blkWriteSpeed msg/sec" >> $1
- echo "Errors : $blkWriteErrors" >> $1
- echo "----------------------------------------------------------------------------------------------" >> $1
-
- echo "READ test metrics |" >> $1
- echo "------------------" >> $1
- echo "Messages: $readMsg" >> $1
- echo "Speed : $readSpeed msg/sec" >> $1
- echo "Errors : $readErrors" >> $1
- echo "----------------------------------------------------------------------------------------------" >> $1
-
- echo "BULK_READ test metrics |" >> $1
- echo "-----------------------" >> $1
- echo "Messages: $blkReadMsg" >> $1
- echo "Speed : $blkReadSpeed msg/sec" >> $1
- echo "Errors : $blkReadErrors" >> $1
- echo "----------------------------------------------------------------------------------------------" >> $1
-
- if [ -n "$writeErrNodes" ]; then
- echo "Nodes having WRITE errors |" >> $1
- echo "-------------------------------" >> $1
- echo "$writeErrNodes" >> $1
- echo "----------------------------------------------------------------------------------------------" >> $1
- fi
-
- if [ -n "$blkWriteErrNodes" ]; then
- echo "Nodes having BULK_WRITE errors |" >> $1
- echo "-------------------------------" >> $1
- echo "$blkWriteErrNodes" >> $1
- echo "----------------------------------------------------------------------------------------------" >> $1
- fi
-
- if [ -n "$readErrNodes" ]; then
- echo "Nodes having READ errors |" >> $1
- echo "-------------------------------" >> $1
- echo "$readErrNodes" >> $1
- echo "----------------------------------------------------------------------------------------------" >> $1
- fi
-
- if [ -n "$blkReadErrNodes" ]; then
- echo "Nodes having BULK_READ errors |" >> $1
- echo "-------------------------------" >> $1
- echo "$blkReadErrNodes" >> $1
- echo "----------------------------------------------------------------------------------------------" >> $1
- fi
-
- cat $tmpFile >> $1
-
- rm -f $tmpFile
-}
-
-# Creates report for failed tests
-reportFailedTestsDetailes()
-{
- for dir in $2/*
- do
- node=$(echo $dir | sed -r "s/^.*\///g")
- if [ -z "$node" ]; then
- continue
- fi
-
- echo "----------------------------------------------------------------------------------------------" >> $1
- echo "Error details for node: $node" >> $1
- echo "----------------------------------------------------------------------------------------------" >> $1
-
- if [ -f "$dir/__error__" ]; then
- cat $dir/__error__ >> $1
- else
- echo "N/A" >> $1
- fi
- done
-}
-
-#######################################################################################################
-
-if [ "$TESTS_TYPE" != "ignite" ] && [ "$TESTS_TYPE" != "cassandra" ]; then
- terminate "Incorrect tests type specified: $TESTS_TYPE"
-fi
-
-reportTestsSummary
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/src/test/java/org/apache/ignite/tests/CassandraDirectPersistenceLoadTest.java
----------------------------------------------------------------------
diff --git a/modules/cassandra/src/test/java/org/apache/ignite/tests/CassandraDirectPersistenceLoadTest.java b/modules/cassandra/src/test/java/org/apache/ignite/tests/CassandraDirectPersistenceLoadTest.java
deleted file mode 100644
index c031bfb..0000000
--- a/modules/cassandra/src/test/java/org/apache/ignite/tests/CassandraDirectPersistenceLoadTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.tests;
-
-import org.apache.ignite.tests.load.LoadTestDriver;
-import org.apache.ignite.tests.load.cassandra.BulkReadWorker;
-import org.apache.ignite.tests.load.cassandra.BulkWriteWorker;
-import org.apache.ignite.tests.load.cassandra.ReadWorker;
-import org.apache.ignite.tests.load.cassandra.WriteWorker;
-import org.apache.ignite.tests.utils.CacheStoreHelper;
-import org.apache.ignite.tests.utils.CassandraHelper;
-import org.apache.ignite.tests.utils.TestsHelper;
-import org.apache.log4j.Logger;
-
-/**
- * Load tests for {@link org.apache.ignite.cache.store.cassandra.CassandraCacheStore} implementation of
- * {@link org.apache.ignite.cache.store.CacheStore} which allows to store Ignite cache data into Cassandra tables.
- */
-public class CassandraDirectPersistenceLoadTest extends LoadTestDriver {
- /** */
- private static final Logger LOGGER = Logger.getLogger("CassandraLoadTests");
-
- /**
- *
- * @param args Test arguments.
- */
- public static void main(String[] args) {
- try {
- LOGGER.info("Cassandra load tests execution started");
-
- LoadTestDriver driver = new CassandraDirectPersistenceLoadTest();
-
- /**
- * Load test scripts could be executed from several machines. Current implementation can correctly,
- * handle situation when Cassandra keyspace/table was dropped - for example by the same load test
- * started a bit later on another machine. Moreover there is a warm up period for each load test.
- * Thus all the delays related to keyspaces/tables recreation actions will not affect performance metrics,
- * but it will be produced lots of "trash" output in the logs (related to correct handling of such
- * exceptional situation and keyspace/table recreation).
- *
- * Thus dropping test keyspaces at the beginning of the tests makes sense only for Unit tests,
- * but not for Load tests.
- **/
-
- //CassandraHelper.dropTestKeyspaces();
-
- driver.runTest("WRITE", WriteWorker.class, WriteWorker.LOGGER_NAME);
-
- driver.runTest("BULK_WRITE", BulkWriteWorker.class, BulkWriteWorker.LOGGER_NAME);
-
- driver.runTest("READ", ReadWorker.class, ReadWorker.LOGGER_NAME);
-
- driver.runTest("BULK_READ", BulkReadWorker.class, BulkReadWorker.LOGGER_NAME);
-
- /**
- * Load test script executed on one machine could complete earlier that the same load test executed from
- * another machine. Current implementation can correctly handle situation when Cassandra keyspace/table
- * was dropped (simply recreate it). But dropping keyspace/table during load tests execution and subsequent
- * recreation of such objects can have SIGNIFICANT EFFECT on final performance metrics.
- *
- * Thus dropping test keyspaces at the end of the tests makes sense only for Unit tests,
- * but not for Load tests.
- */
-
- //CassandraHelper.dropTestKeyspaces(); // REVIEW This line is commented by purpose?
-
- LOGGER.info("Cassandra load tests execution completed");
- }
- catch (Throwable e) {
- LOGGER.error("Cassandra load tests execution failed", e);
- throw new RuntimeException("Cassandra load tests execution failed", e);
- }
- finally {
- CassandraHelper.releaseCassandraResources();
- }
- }
-
- /** {@inheritDoc} */
- @Override protected Logger logger() {
- return LOGGER;
- }
-
- /** {@inheritDoc} */
- @Override protected Object setup(String logName) {
- return CacheStoreHelper.createCacheStore(
- TestsHelper.getLoadTestsCacheName(),
- TestsHelper.getLoadTestsPersistenceSettings(),
- CassandraHelper.getAdminDataSrc(),
- Logger.getLogger(logName));
- }
-
-}