You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tg...@apache.org on 2017/04/12 20:52:44 UTC
[4/6] beam git commit: Move HIFIO k8s scripts into shared dir
Move HIFIO k8s scripts into shared dir
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/34b95076
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/34b95076
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/34b95076
Branch: refs/heads/master
Commit: 34b95076d57baf7af6f7b8b0aaf748868952620b
Parents: 5758899
Author: Stephen Sisk <si...@google.com>
Authored: Fri Apr 7 16:11:19 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Wed Apr 12 13:52:28 2017 -0700
----------------------------------------------------------------------
.../SmallITCluster/cassandra-svc-rc.yaml | 88 ++++++
.../cassandra/SmallITCluster/start-up.sh | 21 ++
.../cassandra/SmallITCluster/teardown.sh | 21 ++
.../kubernetes/cassandra/data-load-setup.sh | 29 ++
.test-infra/kubernetes/cassandra/data-load.sh | 67 +++++
.../LargeProductionCluster/es-services.yaml | 277 +++++++++++++++++
.../LargeProductionCluster/start-up.sh | 21 ++
.../LargeProductionCluster/teardown.sh | 20 ++
.../SmallITCluster/elasticsearch-svc-rc.yaml | 84 ++++++
.../elasticsearch/SmallITCluster/start-up.sh | 22 ++
.../elasticsearch/SmallITCluster/teardown.sh | 20 ++
.../kubernetes/elasticsearch/data-load-setup.sh | 26 ++
.../kubernetes/elasticsearch/data-load.sh | 33 ++
.../kubernetes/elasticsearch/es_test_data.py | 299 +++++++++++++++++++
.../kubernetes/elasticsearch/show-health.sh | 25 ++
.../SmallITCluster/cassandra-svc-rc.yaml | 88 ------
.../cassandra/SmallITCluster/start-up.sh | 21 --
.../cassandra/SmallITCluster/teardown.sh | 21 --
.../kubernetes/cassandra/data-load-setup.sh | 29 --
.../resources/kubernetes/cassandra/data-load.sh | 67 -----
.../LargeProductionCluster/es-services.yaml | 277 -----------------
.../LargeProductionCluster/start-up.sh | 21 --
.../LargeProductionCluster/teardown.sh | 20 --
.../SmallITCluster/elasticsearch-svc-rc.yaml | 84 ------
.../elasticsearch/SmallITCluster/start-up.sh | 22 --
.../elasticsearch/SmallITCluster/teardown.sh | 20 --
.../kubernetes/elasticsearch/data-load-setup.sh | 26 --
.../kubernetes/elasticsearch/data-load.sh | 33 --
.../kubernetes/elasticsearch/es_test_data.py | 299 -------------------
.../kubernetes/elasticsearch/show-health.sh | 25 --
30 files changed, 1053 insertions(+), 1053 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/cassandra/SmallITCluster/cassandra-svc-rc.yaml
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/cassandra/SmallITCluster/cassandra-svc-rc.yaml b/.test-infra/kubernetes/cassandra/SmallITCluster/cassandra-svc-rc.yaml
new file mode 100644
index 0000000..7c36e34
--- /dev/null
+++ b/.test-infra/kubernetes/cassandra/SmallITCluster/cassandra-svc-rc.yaml
@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Headless service that allows us to get the IP addresses of our Cassandra nodes
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ name: cassandra-peers
+ name: cassandra-peers
+spec:
+ clusterIP: None
+ ports:
+ - port: 7000
+ name: intra-node-communication
+ - port: 7001
+ name: tls-intra-node-communication
+ selector:
+ name: cassandra
+---
+# Kubernetes service file exposing Cassandra endpoint used by clients.
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ name: cassandra
+ name: cassandra
+spec:
+ ports:
+ - port: 9042
+ name: cql
+ selector:
+ name: cassandra
+ type: LoadBalancer
+---
+# Replication Controller for Cassandra which tracks the Cassandra pods.
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ labels:
+ name: cassandra
+ name: cassandra
+spec:
+ replicas: 1
+ selector:
+ name: cassandra
+ template:
+ metadata:
+ labels:
+ name: cassandra
+ spec:
+ containers:
+ - image: cassandra
+ name: cassandra
+ env:
+ - name: PEER_DISCOVERY_SERVICE
+ value: cassandra-peers
+ - name: CASSANDRA_CLUSTER_NAME
+ value: Cassandra
+ - name: CASSANDRA_DC
+ value: DC1
+ - name: CASSANDRA_RACK
+ value: Kubernetes Cluster
+# Number of tokens currently configured to 1. If this is not configured, default value is 256. You can change it as per requirement.
+ - name: CASSANDRA_NUM_TOKENS
+ value: '1'
+ ports:
+ - containerPort: 9042
+ name: cql
+ volumeMounts:
+ - mountPath: /var/lib/cassandra/data
+ name: data
+ volumes:
+ - name: data
+ emptyDir: {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/cassandra/SmallITCluster/start-up.sh
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/cassandra/SmallITCluster/start-up.sh b/.test-infra/kubernetes/cassandra/SmallITCluster/start-up.sh
new file mode 100644
index 0000000..c05b771
--- /dev/null
+++ b/.test-infra/kubernetes/cassandra/SmallITCluster/start-up.sh
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+set -e
+
+# Create Cassandra services and Replication controller.
+kubectl create -f cassandra-svc-rc.yaml
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/cassandra/SmallITCluster/teardown.sh
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/cassandra/SmallITCluster/teardown.sh b/.test-infra/kubernetes/cassandra/SmallITCluster/teardown.sh
new file mode 100644
index 0000000..f538a75
--- /dev/null
+++ b/.test-infra/kubernetes/cassandra/SmallITCluster/teardown.sh
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#!/bin/bash
+set -e
+
+# Delete Cassandra services and Replication controller.
+kubectl delete -f cassandra-svc-rc.yaml
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/cassandra/data-load-setup.sh
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/cassandra/data-load-setup.sh b/.test-infra/kubernetes/cassandra/data-load-setup.sh
new file mode 100644
index 0000000..4e12f89
--- /dev/null
+++ b/.test-infra/kubernetes/cassandra/data-load-setup.sh
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+set -e
+
+# Load YCSB tool
+echo "Downloading YCSB tool"
+echo "------------------------------"
+curl -O --location https://github.com/brianfrankcooper/YCSB/releases/download/0.12.0/ycsb-0.12.0.tar.gz
+tar xfz ycsb-0.12.0.tar.gz
+wget https://www.slf4j.org/dist/slf4j-1.7.22.tar.gz
+tar xfz slf4j-1.7.22.tar.gz
+cp slf4j-1.7.22/slf4j-simple-*.jar ycsb-0.12.0/lib/
+cp slf4j-1.7.22/slf4j-api-*.jar ycsb-0.12.0/lib/
+echo "YCSB tool loaded"
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/cassandra/data-load.sh
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/cassandra/data-load.sh b/.test-infra/kubernetes/cassandra/data-load.sh
new file mode 100644
index 0000000..59d0e22
--- /dev/null
+++ b/.test-infra/kubernetes/cassandra/data-load.sh
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+set -e
+
+recordcount=1000
+# Identify the pod
+cassandra_pods="kubectl get pods -l name=cassandra"
+running_seed="$(kubectl get pods -o json -l name=cassandra -o jsonpath=\
+'{.items[0].metadata.name}')"
+echo "Detected Running Pod $running_seed"
+
+# After starting the service, it takes couple of minutes to generate the external IP for the
+# service. Hence, wait for sometime.
+
+# Identify external IP of the pod
+external_ip="$(kubectl get svc cassandra -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
+echo "Waiting for the Cassandra service to come up ........"
+while [ -z "$external_ip" ]
+do
+ sleep 10s
+ external_ip="$(kubectl get svc cassandra -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
+ echo "."
+done
+echo "External IP - $external_ip"
+
+# Create keyspace
+keyspace_creation_command="drop keyspace if exists ycsb;create keyspace ycsb WITH REPLICATION = {\
+'class' : 'SimpleStrategy', 'replication_factor': 3 };"
+kubectl exec -ti $running_seed -- cqlsh -e "$keyspace_creation_command"
+echo "Keyspace creation............"
+echo "-----------------------------"
+echo "$keyspace_creation_command"
+echo
+
+# Create table
+table_creation_command="use ycsb;drop table if exists usertable;create table usertable (\
+y_id varchar primary key,field0 varchar,field1 varchar,field2 varchar,field3 varchar,\
+field4 varchar,field5 varchar,field6 varchar,field7 varchar,field8 varchar,field9 varchar);"
+kubectl exec -ti $running_seed -- cqlsh -e "$table_creation_command"
+echo "Table creation .............."
+echo "-----------------------------"
+echo "$table_creation_command"
+
+cd ycsb-0.12.0
+
+echo "Starting to load data on ${external_ip}"
+echo "-----------------------------"
+# Record count set to 1000, change this value to load as per requirement.
+# dataintegrity flag is set to true to load deterministic data
+./bin/ycsb load cassandra-cql -p hosts=${external_ip} -p dataintegrity=true -p recordcount=\
+${recordcount} -p insertorder=ordered -p fieldlength=20 -P workloads/workloadd \
+-s > workloada_load_res.txt
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/es-services.yaml
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/es-services.yaml b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/es-services.yaml
new file mode 100644
index 0000000..38c820e
--- /dev/null
+++ b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/es-services.yaml
@@ -0,0 +1,277 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Service file containing services for ES discovery, elasticsearch and master node deployment.
+
+# Kubernetes headless service for Elasticsearch discovery of nodes.
+apiVersion: v1
+kind: Service
+metadata:
+ name: elasticsearch-discovery
+ labels:
+ component: elasticsearch
+ role: master
+spec:
+ selector:
+ component: elasticsearch
+ role: master
+ ports:
+ - name: transport
+ port: 9300
+ protocol: TCP
+---
+# To create Elasticsearch frontend cluster Kubernetes service.
+# It sets up a load balancer on TCP port 9200 that distributes network traffic to the ES client nodes.
+apiVersion: v1
+kind: Service
+metadata:
+ name: elasticsearch
+ labels:
+ component: elasticsearch
+ role: client
+spec:
+ type: LoadBalancer
+ selector:
+ component: elasticsearch
+ role: client
+ ports:
+ - name: http
+ port: 9200
+ protocol: TCP
+---
+# The Kubernetes deployment script for Elasticsearch master nodes.
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: es-master
+ labels:
+ component: elasticsearch
+ role: master
+spec:
+ replicas: 3
+ template:
+ metadata:
+ labels:
+ component: elasticsearch
+ role: master
+ annotations:
+ pod.beta.kubernetes.io/init-containers: '[
+ {
+ "name": "sysctl",
+ "image": "busybox",
+ "imagePullPolicy": "IfNotPresent",
+ "command": ["sysctl", "-w", "vm.max_map_count=262144"],
+ "securityContext": {
+ "privileged": true
+ }
+ }
+ ]'
+ spec:
+ containers:
+ - name: es-master
+ securityContext:
+ privileged: false
+ capabilities:
+ add:
+# IPC_LOCK capability is enabled to allow Elasticsearch to lock the heap in memory so it will not be swapped.
+ - IPC_LOCK
+# SYS_RESOURCE is docker capability key to control and override the resource limits.
+# This could be needed to increase base limits.(e.g. File descriptor limit for elasticsearch)
+ - SYS_RESOURCE
+ image: quay.io/pires/docker-elasticsearch-kubernetes:5.2.2
+ env:
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: "CLUSTER_NAME"
+ value: "myesdb"
+ - name: "NUMBER_OF_MASTERS"
+ value: "2"
+ - name: NODE_MASTER
+ value: "true"
+ - name: NODE_INGEST
+ value: "false"
+ - name: NODE_DATA
+ value: "false"
+ - name: HTTP_ENABLE
+ value: "false"
+ - name: "ES_JAVA_OPTS"
+ value: "-Xms256m -Xmx256m"
+ ports:
+ - containerPort: 9300
+ name: transport
+ protocol: TCP
+ volumeMounts:
+ - name: storage
+ mountPath: /data
+ volumes:
+ - emptyDir:
+ medium: ""
+ name: "storage"
+---
+# Kubernetes deployment script for Elasticsearch client nodes (aka load balancing proxies).
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: es-client
+ labels:
+ component: elasticsearch
+ role: client
+spec:
+ # The no. of replicas can be incremented based on the client usage using HTTP API.
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ component: elasticsearch
+ role: client
+ annotations:
+ # Elasticsearch uses a hybrid mmapfs / niofs directory by default to store its indices.
+ # The default operating system limits on mmap counts is likely to be too low, which may result
+ # in out of memory exceptions. Therefore, the need to increase virtual memory
+ # vm.max_map_count for large amount of data in the pod initialization annotation.
+ pod.beta.kubernetes.io/init-containers: '[
+ {
+ "name": "sysctl",
+ "image": "busybox",
+ "imagePullPolicy": "IfNotPresent",
+ "command": ["sysctl", "-w", "vm.max_map_count=262144"],
+ "securityContext": {
+ "privileged": true
+ }
+ }
+ ]'
+ spec:
+ containers:
+ - name: es-client
+ securityContext:
+ privileged: false
+ capabilities:
+ add:
+# IPC_LOCK capability is enabled to allow Elasticsearch to lock the heap in memory so it will not be swapped.
+ - IPC_LOCK
+# SYS_RESOURCE is docker capability key to control and override the resource limits.
+# This could be needed to increase base limits.(e.g. File descriptor limit for elasticsearch)
+ - SYS_RESOURCE
+ image: quay.io/pires/docker-elasticsearch-kubernetes:5.2.2
+ env:
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: "CLUSTER_NAME"
+ value: "myesdb"
+ - name: NODE_MASTER
+ value: "false"
+ - name: NODE_DATA
+ value: "false"
+ - name: HTTP_ENABLE
+ value: "true"
+ - name: "ES_JAVA_OPTS"
+ value: "-Xms256m -Xmx256m"
+ ports:
+ - containerPort: 9200
+ name: http
+ protocol: TCP
+ - containerPort: 9300
+ name: transport
+ protocol: TCP
+ volumeMounts:
+ - name: storage
+ mountPath: /data
+ volumes:
+ - emptyDir:
+ medium: ""
+ name: "storage"
+---
+# Kubernetes deployment script for Elasticsearch data nodes which store and index data.
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: es-data
+ labels:
+ component: elasticsearch
+ role: data
+spec:
+ replicas: 2
+ template:
+ metadata:
+ labels:
+ component: elasticsearch
+ role: data
+ annotations:
+ pod.beta.kubernetes.io/init-containers: '[
+ {
+ "name": "sysctl",
+ "image": "busybox",
+ "imagePullPolicy": "IfNotPresent",
+ "command": ["sysctl", "-w", "vm.max_map_count=1048575"],
+ "securityContext": {
+ "privileged": true
+ }
+ }
+ ]'
+ spec:
+ containers:
+ - name: es-data
+ securityContext:
+ privileged: false
+ capabilities:
+ add:
+# IPC_LOCK capability is enabled to allow Elasticsearch to lock the heap in memory so it will not be swapped.
+ - IPC_LOCK
+# SYS_RESOURCE is docker capability key to control and override the resource limits.
+# This could be needed to increase base limits.(e.g. File descriptor limit for elasticsearch)
+ - SYS_RESOURCE
+ image: quay.io/pires/docker-elasticsearch-kubernetes:5.2.2
+ env:
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: "CLUSTER_NAME"
+ value: "myesdb"
+ - name: NODE_MASTER
+ value: "false"
+ - name: NODE_INGEST
+ value: "false"
+ - name: HTTP_ENABLE
+ value: "false"
+ - name: "ES_JAVA_OPTS"
+ value: "-Xms256m -Xmx256m"
+ ports:
+ - containerPort: 9300
+ name: transport
+ protocol: TCP
+ volumeMounts:
+ - name: storage
+ mountPath: /data
+ volumes:
+ - emptyDir:
+ medium: ""
+ name: "storage"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh
new file mode 100644
index 0000000..4d277c8
--- /dev/null
+++ b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#!/bin/sh
+set -e
+
+# Create Elasticsearch services and deployments.
+kubectl create -f es-services.yaml
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh
new file mode 100644
index 0000000..a30793b
--- /dev/null
+++ b/.test-infra/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+set -e
+
+# Delete elasticsearch services and deployments.
+kubectl delete -f es-services.yaml
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/elasticsearch/SmallITCluster/elasticsearch-svc-rc.yaml
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/elasticsearch/SmallITCluster/elasticsearch-svc-rc.yaml b/.test-infra/kubernetes/elasticsearch/SmallITCluster/elasticsearch-svc-rc.yaml
new file mode 100644
index 0000000..9a7ac3d
--- /dev/null
+++ b/.test-infra/kubernetes/elasticsearch/SmallITCluster/elasticsearch-svc-rc.yaml
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# To create Elasticsearch frontend cluster Kubernetes service.
+# It sets up a load balancer on TCP port 9200 that distributes network traffic to the ES nodes.
+apiVersion: v1
+kind: Service
+metadata:
+ name: elasticsearch
+ labels:
+ component: elasticsearch
+spec:
+ type: LoadBalancer
+ selector:
+ component: elasticsearch
+ ports:
+ - name: http
+ port: 9200
+ protocol: TCP
+ - name: transport
+ port: 9300
+ protocol: TCP
+---
+# The Kubernetes deployment script for Elasticsearch replication nodes. It will create 1 node cluster.
+# To scale the cluster as desired, you can create replicas of node use 'kubectl scale --replicas=3 rc es' command
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: es
+ labels:
+ component: elasticsearch
+spec:
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ component: elasticsearch
+ spec:
+ containers:
+ - name: es
+ securityContext:
+ capabilities:
+ add:
+# IPC_LOCK capability is enabled to allow Elasticsearch to lock the heap in memory so it will not be swapped.
+ - IPC_LOCK
+# SYS_RESOURCE capability is set to control and override various resource limits.
+ - SYS_RESOURCE
+ image: quay.io/pires/docker-elasticsearch-kubernetes:5.2.2
+ env:
+ - name: "CLUSTER_NAME"
+ value: "myesdb"
+ - name: "DISCOVERY_SERVICE"
+ value: "elasticsearch"
+ - name: NODE_MASTER
+ value: "true"
+ - name: NODE_DATA
+ value: "true"
+ - name: HTTP_ENABLE
+ value: "true"
+ ports:
+ - containerPort: 9200
+ name: http
+ protocol: TCP
+ - containerPort: 9300
+ name: transport
+ protocol: TCP
+ volumeMounts:
+ - mountPath: /data
+ name: storage
+ volumes:
+ - name: storage
+ emptyDir: {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/elasticsearch/SmallITCluster/start-up.sh
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/elasticsearch/SmallITCluster/start-up.sh b/.test-infra/kubernetes/elasticsearch/SmallITCluster/start-up.sh
new file mode 100644
index 0000000..e8cf275
--- /dev/null
+++ b/.test-infra/kubernetes/elasticsearch/SmallITCluster/start-up.sh
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#!/bin/sh
+set -e
+
+# Create Elasticsearch services and deployments.
+kubectl create -f elasticsearch-svc-rc.yaml
+
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/elasticsearch/SmallITCluster/teardown.sh
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/elasticsearch/SmallITCluster/teardown.sh b/.test-infra/kubernetes/elasticsearch/SmallITCluster/teardown.sh
new file mode 100644
index 0000000..079141d
--- /dev/null
+++ b/.test-infra/kubernetes/elasticsearch/SmallITCluster/teardown.sh
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+set -e
+
+# Delete elasticsearch services and deployments.
+kubectl delete -f elasticsearch-svc-rc.yaml
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/elasticsearch/data-load-setup.sh
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/elasticsearch/data-load-setup.sh b/.test-infra/kubernetes/elasticsearch/data-load-setup.sh
new file mode 100644
index 0000000..00991bc
--- /dev/null
+++ b/.test-infra/kubernetes/elasticsearch/data-load-setup.sh
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+set -e
+
+# Install python
+sudo apt-get update
+sudo apt-get install python-pip
+sudo pip install --upgrade pip
+sudo apt-get install python-dev
+sudo pip install tornado numpy
+echo
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/elasticsearch/data-load.sh
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/elasticsearch/data-load.sh b/.test-infra/kubernetes/elasticsearch/data-load.sh
new file mode 100644
index 0000000..21150fb
--- /dev/null
+++ b/.test-infra/kubernetes/elasticsearch/data-load.sh
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+set -e
+
+# Identify external IP
+external_ip="$(kubectl get svc elasticsearch -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
+echo "Waiting for the Elasticsearch service to come up ........"
+while [ -z "$external_ip" ]
+do
+ sleep 10s
+ external_ip="$(kubectl get svc elasticsearch -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
+ echo "."
+done
+echo "External IP - $external_ip"
+echo
+
+# Run the script
+/usr/bin/python es_test_data.py --count=1000 --format=Txn_ID:int,Item_Code:int,Item_ID:int,User_Name:str,last_updated:ts,Price:int,Title:str,Description:str,Age:int,Item_Name:str,Item_Price:int,Availability:bool,Batch_Num:int,Last_Ordered:tstxt,City:text --es_url=http://$external_ip:9200 &
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/elasticsearch/es_test_data.py
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/elasticsearch/es_test_data.py b/.test-infra/kubernetes/elasticsearch/es_test_data.py
new file mode 100644
index 0000000..1658e2c
--- /dev/null
+++ b/.test-infra/kubernetes/elasticsearch/es_test_data.py
@@ -0,0 +1,299 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script to populate data on Elasticsearch
+# Hashcode for 1000 records is ed36c09b5e24a95fd8d3cc711a043a85320bb47d,
+# For test with query to select one record from 1000 docs,
+# hashcode is 83c108ff81e87b6f3807c638e6bb9a9e3d430dc7
+# Hashcode for 50m records (~20 gigs) is aff7390ee25c4c330f0a58dfbfe335421b11e405
+#!/usr/bin/python
+
+import json
+import time
+import logging
+import random
+import string
+import uuid
+import datetime
+
+import tornado.gen
+import tornado.httpclient
+import tornado.ioloop
+import tornado.options
+
+async_http_client = tornado.httpclient.AsyncHTTPClient()
+id_counter = 0
+upload_data_count = 0
+_dict_data = None
+
+
+
+def delete_index(idx_name):
+ try:
+ url = "%s/%s?refresh=true" % (tornado.options.options.es_url, idx_name)
+ request = tornado.httpclient.HTTPRequest(url, method="DELETE", request_timeout=240,
+ auth_username=tornado.options.options.username,
+ auth_password=tornado.options.options.password)
+ response = tornado.httpclient.HTTPClient().fetch(request)
+ logging.info('Deleting index "%s" done %s' % (idx_name, response.body))
+ except tornado.httpclient.HTTPError:
+ pass
+
+
+def create_index(idx_name):
+ schema = {
+ "settings": {
+ "number_of_shards": tornado.options.options.num_of_shards,
+ "number_of_replicas": tornado.options.options.num_of_replicas
+ },
+ "refresh": True
+ }
+
+ body = json.dumps(schema)
+ url = "%s/%s" % (tornado.options.options.es_url, idx_name)
+ try:
+ logging.info('Trying to create index %s' % (url))
+ request = tornado.httpclient.HTTPRequest(url, method="PUT", body=body, request_timeout=240,
+ auth_username=tornado.options.options.username,
+ auth_password=tornado.options.options.password)
+ response = tornado.httpclient.HTTPClient().fetch(request)
+ logging.info('Creating index "%s" done %s' % (idx_name, response.body))
+ except tornado.httpclient.HTTPError:
+ logging.info('Looks like the index exists already')
+ pass
+
+
+@tornado.gen.coroutine
+def upload_batch(upload_data_txt):
+ try:
+ request = tornado.httpclient.HTTPRequest(tornado.options.options.es_url + "/_bulk",
+ method="POST", body=upload_data_txt,
+ request_timeout=
+ tornado.options.options.http_upload_timeout,
+ auth_username=tornado.options.options.username,
+ auth_password=tornado.options.options.password)
+ response = yield async_http_client.fetch(request)
+ except Exception as ex:
+ logging.error("upload failed, error: %s" % ex)
+ return
+
+ result = json.loads(response.body.decode('utf-8'))
+ res_txt = "OK" if not result['errors'] else "FAILED"
+ took = int(result['took'])
+ logging.info("Upload: %s - upload took: %5dms, total docs uploaded: %7d" % (res_txt, took,
+ upload_data_count))
+
+
+def get_data_for_format(format,count):
+ split_f = format.split(":")
+ if not split_f:
+ return None, None
+
+ field_name = split_f[0]
+ field_type = split_f[1]
+
+ return_val = ''
+
+ if field_type == "bool":
+ if count%2 == 0:
+ return_val = True
+ else:
+ return_val = False
+
+ elif field_type == "str":
+ return_val = field_name + str(count)
+
+ elif field_type == "int":
+ return_val = count
+
+ elif field_type == "ipv4":
+ return_val = "{0}.{1}.{2}.{3}".format(1,2,3,count%255)
+
+ elif field_type in ["ts", "tstxt"]:
+ return_val = int(count * 1000) if field_type == "ts" else\
+ datetime.datetime.fromtimestamp(count)\
+ .strftime("%Y-%m-%dT%H:%M:%S.000-0000")
+
+ elif field_type == "words":
+ return_val = field_name + str(count)
+
+ elif field_type == "dict":
+ mydict = dict(a=field_name + str(count), b=field_name + str(count), c=field_name + str(count),
+ d=field_name + str(count), e=field_name + str(count), f=field_name + str(count),
+ g=field_name + str(count), h=field_name + str(count), i=field_name + str(count),
+ j=field_name + str(count))
+ return_val = ", ".join("=".join(_) for _ in mydict.items())
+
+ elif field_type == "text":
+ return_val = field_name + str(count)
+
+ return field_name, return_val
+
+
+def generate_count(min, max):
+ if min == max:
+ return max
+ elif min > max:
+ return random.randrange(max, min);
+ else:
+ return random.randrange(min, max);
+
+
+def generate_random_doc(format,count):
+ global id_counter
+
+ res = {}
+
+ for f in format:
+ f_key, f_val = get_data_for_format(f,count)
+ if f_key:
+ res[f_key] = f_val
+
+ if not tornado.options.options.id_type:
+ return res
+
+ if tornado.options.options.id_type == 'int':
+ res['_id'] = id_counter
+ id_counter += 1
+ elif tornado.options.options.id_type == 'uuid4':
+ res['_id'] = str(uuid.uuid4())
+
+ return res
+
+
+def set_index_refresh(val):
+
+ params = {"index": {"refresh_interval": val}}
+ body = json.dumps(params)
+ url = "%s/%s/_settings" % (tornado.options.options.es_url, tornado.options.options.index_name)
+ try:
+ request = tornado.httpclient.HTTPRequest(url, method="PUT", body=body, request_timeout=240,
+ auth_username=tornado.options.options.username,
+ auth_password=tornado.options.options.password)
+ http_client = tornado.httpclient.HTTPClient()
+ http_client.fetch(request)
+ logging.info('Set index refresh to %s' % val)
+ except Exception as ex:
+ logging.exception(ex)
+
+
+@tornado.gen.coroutine
+def generate_test_data():
+
+ global upload_data_count
+
+ if tornado.options.options.force_init_index:
+ delete_index(tornado.options.options.index_name)
+
+ create_index(tornado.options.options.index_name)
+
+ # todo: query what refresh is set to, then restore later
+ if tornado.options.options.set_refresh:
+ set_index_refresh("-1")
+
+ if tornado.options.options.out_file:
+ out_file = open(tornado.options.options.out_file, "w")
+ else:
+ out_file = None
+
+ if tornado.options.options.dict_file:
+ global _dict_data
+ with open(tornado.options.options.dict_file, 'r') as f:
+ _dict_data = f.readlines()
+ logging.info("Loaded %d words from the %s" % (len(_dict_data),
+ tornado.options.options.dict_file))
+
+ format = tornado.options.options.format.split(',')
+ if not format:
+ logging.error('invalid format')
+ exit(1)
+
+ ts_start = int(time.time())
+ upload_data_txt = ""
+ total_uploaded = 0
+
+ logging.info("Generating %d docs, upload batch size is %d" % (tornado.options.options.count,
+ tornado.options
+ .options.batch_size))
+ for num in range(0, tornado.options.options.count):
+
+ item = generate_random_doc(format,num)
+
+ if out_file:
+ out_file.write("%s\n" % json.dumps(item))
+
+ cmd = {'index': {'_index': tornado.options.options.index_name,
+ '_type': tornado.options.options.index_type}}
+ if '_id' in item:
+ cmd['index']['_id'] = item['_id']
+
+ upload_data_txt += json.dumps(cmd) + "\n"
+ upload_data_txt += json.dumps(item) + "\n"
+ upload_data_count += 1
+
+ if upload_data_count % tornado.options.options.batch_size == 0:
+ yield upload_batch(upload_data_txt)
+ upload_data_txt = ""
+
+ # upload remaining items in `upload_data_txt`
+ if upload_data_txt:
+ yield upload_batch(upload_data_txt)
+
+ if tornado.options.options.set_refresh:
+ set_index_refresh("1s")
+
+ if out_file:
+ out_file.close()
+
+ took_secs = int(time.time() - ts_start)
+
+ logging.info("Done - total docs uploaded: %d, took %d seconds" %
+ (tornado.options.options.count, took_secs))
+
+
+if __name__ == '__main__':
+ tornado.options.define("es_url", type=str, default='http://localhost:9200/',
+ help="URL of your Elasticsearch node")
+ tornado.options.define("index_name", type=str, default='test_data',
+ help="Name of the index to store your messages")
+ tornado.options.define("index_type", type=str, default='test_type', help="Type")
+ tornado.options.define("batch_size", type=int, default=1000,
+ help="Elasticsearch bulk index batch size")
+ tornado.options.define("num_of_shards", type=int, default=2,
+ help="Number of shards for ES index")
+ tornado.options.define("http_upload_timeout", type=int, default=3,
+ help="Timeout in seconds when uploading data")
+ tornado.options.define("count", type=int, default=100000, help="Number of docs to generate")
+ tornado.options.define("format", type=str, default='name:str,age:int,last_updated:ts',
+ help="message format")
+ tornado.options.define("num_of_replicas", type=int, default=0,
+ help="Number of replicas for ES index")
+ tornado.options.define("force_init_index", type=bool, default=False,
+ help="Force deleting and re-initializing the Elasticsearch index")
+ tornado.options.define("set_refresh", type=bool, default=False,
+ help="Set refresh rate to -1 before starting the upload")
+ tornado.options.define("out_file", type=str, default=False,
+ help="If set, write test data to out_file as well.")
+ tornado.options.define("id_type", type=str, default=None,
+ help="Type of 'id' to use for the docs, \
+ valid settings are int and uuid4, None is default")
+ tornado.options.define("dict_file", type=str, default=None,
+ help="Name of dictionary file to use")
+ tornado.options.define("username", type=str, default=None, help="Username for elasticsearch")
+ tornado.options.define("password", type=str, default=None, help="Password for elasticsearch")
+ tornado.options.parse_command_line()
+
+ tornado.ioloop.IOLoop.instance().run_sync(generate_test_data)
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/.test-infra/kubernetes/elasticsearch/show-health.sh
----------------------------------------------------------------------
diff --git a/.test-infra/kubernetes/elasticsearch/show-health.sh b/.test-infra/kubernetes/elasticsearch/show-health.sh
new file mode 100644
index 0000000..8fa912c
--- /dev/null
+++ b/.test-infra/kubernetes/elasticsearch/show-health.sh
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/sh
+set -e
+
+external_ip="$(kubectl get svc elasticsearch -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
+
+echo "Elasticsearch cluster health info"
+echo "---------------------------------"
+curl $external_ip:9200/_cluster/health
+echo # empty line since curl doesn't output CRLF
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/SmallITCluster/cassandra-svc-rc.yaml
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/SmallITCluster/cassandra-svc-rc.yaml b/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/SmallITCluster/cassandra-svc-rc.yaml
deleted file mode 100644
index 7c36e34..0000000
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/SmallITCluster/cassandra-svc-rc.yaml
+++ /dev/null
@@ -1,88 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Headless service that allows us to get the IP addresses of our Cassandra nodes
-apiVersion: v1
-kind: Service
-metadata:
- labels:
- name: cassandra-peers
- name: cassandra-peers
-spec:
- clusterIP: None
- ports:
- - port: 7000
- name: intra-node-communication
- - port: 7001
- name: tls-intra-node-communication
- selector:
- name: cassandra
----
-# Kubernetes service file exposing Cassandra endpoint used by clients.
-apiVersion: v1
-kind: Service
-metadata:
- labels:
- name: cassandra
- name: cassandra
-spec:
- ports:
- - port: 9042
- name: cql
- selector:
- name: cassandra
- type: LoadBalancer
----
-# Replication Controller for Cassandra which tracks the Cassandra pods.
-apiVersion: v1
-kind: ReplicationController
-metadata:
- labels:
- name: cassandra
- name: cassandra
-spec:
- replicas: 1
- selector:
- name: cassandra
- template:
- metadata:
- labels:
- name: cassandra
- spec:
- containers:
- - image: cassandra
- name: cassandra
- env:
- - name: PEER_DISCOVERY_SERVICE
- value: cassandra-peers
- - name: CASSANDRA_CLUSTER_NAME
- value: Cassandra
- - name: CASSANDRA_DC
- value: DC1
- - name: CASSANDRA_RACK
- value: Kubernetes Cluster
-# Number of tokens currently configured to 1. If this is not configured, default value is 256. You can change it as per requirement.
- - name: CASSANDRA_NUM_TOKENS
- value: '1'
- ports:
- - containerPort: 9042
- name: cql
- volumeMounts:
- - mountPath: /var/lib/cassandra/data
- name: data
- volumes:
- - name: data
- emptyDir: {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/SmallITCluster/start-up.sh
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/SmallITCluster/start-up.sh b/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/SmallITCluster/start-up.sh
deleted file mode 100644
index c05b771..0000000
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/SmallITCluster/start-up.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/bin/bash
-set -e
-
-# Create Cassandra services and Replication controller.
-kubectl create -f cassandra-svc-rc.yaml
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/SmallITCluster/teardown.sh
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/SmallITCluster/teardown.sh b/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/SmallITCluster/teardown.sh
deleted file mode 100644
index f538a75..0000000
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/SmallITCluster/teardown.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#!/bin/bash
-set -e
-
-# Delete Cassandra services and Replication controller.
-kubectl delete -f cassandra-svc-rc.yaml
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/data-load-setup.sh
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/data-load-setup.sh b/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/data-load-setup.sh
deleted file mode 100644
index 4e12f89..0000000
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/data-load-setup.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/bin/bash
-set -e
-
-# Load YCSB tool
-echo "Downloading YCSB tool"
-echo "------------------------------"
-curl -O --location https://github.com/brianfrankcooper/YCSB/releases/download/0.12.0/ycsb-0.12.0.tar.gz
-tar xfz ycsb-0.12.0.tar.gz
-wget https://www.slf4j.org/dist/slf4j-1.7.22.tar.gz
-tar xfz slf4j-1.7.22.tar.gz
-cp slf4j-1.7.22/slf4j-simple-*.jar ycsb-0.12.0/lib/
-cp slf4j-1.7.22/slf4j-api-*.jar ycsb-0.12.0/lib/
-echo "YCSB tool loaded"
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/data-load.sh
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/data-load.sh b/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/data-load.sh
deleted file mode 100644
index 59d0e22..0000000
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/cassandra/data-load.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/bin/bash
-set -e
-
-recordcount=1000
-# Identify the pod
-cassandra_pods="kubectl get pods -l name=cassandra"
-running_seed="$(kubectl get pods -o json -l name=cassandra -o jsonpath=\
-'{.items[0].metadata.name}')"
-echo "Detected Running Pod $running_seed"
-
-# After starting the service, it takes couple of minutes to generate the external IP for the
-# service. Hence, wait for sometime.
-
-# Identify external IP of the pod
-external_ip="$(kubectl get svc cassandra -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
-echo "Waiting for the Cassandra service to come up ........"
-while [ -z "$external_ip" ]
-do
- sleep 10s
- external_ip="$(kubectl get svc cassandra -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
- echo "."
-done
-echo "External IP - $external_ip"
-
-# Create keyspace
-keyspace_creation_command="drop keyspace if exists ycsb;create keyspace ycsb WITH REPLICATION = {\
-'class' : 'SimpleStrategy', 'replication_factor': 3 };"
-kubectl exec -ti $running_seed -- cqlsh -e "$keyspace_creation_command"
-echo "Keyspace creation............"
-echo "-----------------------------"
-echo "$keyspace_creation_command"
-echo
-
-# Create table
-table_creation_command="use ycsb;drop table if exists usertable;create table usertable (\
-y_id varchar primary key,field0 varchar,field1 varchar,field2 varchar,field3 varchar,\
-field4 varchar,field5 varchar,field6 varchar,field7 varchar,field8 varchar,field9 varchar);"
-kubectl exec -ti $running_seed -- cqlsh -e "$table_creation_command"
-echo "Table creation .............."
-echo "-----------------------------"
-echo "$table_creation_command"
-
-cd ycsb-0.12.0
-
-echo "Starting to load data on ${external_ip}"
-echo "-----------------------------"
-# Record count set to 1000, change this value to load as per requirement.
-# dataintegrity flag is set to true to load deterministic data
-./bin/ycsb load cassandra-cql -p hosts=${external_ip} -p dataintegrity=true -p recordcount=\
-${recordcount} -p insertorder=ordered -p fieldlength=20 -P workloads/workloadd \
--s > workloada_load_res.txt
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/LargeProductionCluster/es-services.yaml
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/LargeProductionCluster/es-services.yaml b/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/LargeProductionCluster/es-services.yaml
deleted file mode 100644
index 38c820e..0000000
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/LargeProductionCluster/es-services.yaml
+++ /dev/null
@@ -1,277 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Service file containing services for ES discovery, elasticsearch and master node deployment.
-
-# Kubernetes headless service for Elasticsearch discovery of nodes.
-apiVersion: v1
-kind: Service
-metadata:
- name: elasticsearch-discovery
- labels:
- component: elasticsearch
- role: master
-spec:
- selector:
- component: elasticsearch
- role: master
- ports:
- - name: transport
- port: 9300
- protocol: TCP
----
-# To create Elasticsearch frontend cluster Kubernetes service.
-# It sets up a load balancer on TCP port 9200 that distributes network traffic to the ES client nodes.
-apiVersion: v1
-kind: Service
-metadata:
- name: elasticsearch
- labels:
- component: elasticsearch
- role: client
-spec:
- type: LoadBalancer
- selector:
- component: elasticsearch
- role: client
- ports:
- - name: http
- port: 9200
- protocol: TCP
----
-# The Kubernetes deployment script for Elasticsearch master nodes.
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
- name: es-master
- labels:
- component: elasticsearch
- role: master
-spec:
- replicas: 3
- template:
- metadata:
- labels:
- component: elasticsearch
- role: master
- annotations:
- pod.beta.kubernetes.io/init-containers: '[
- {
- "name": "sysctl",
- "image": "busybox",
- "imagePullPolicy": "IfNotPresent",
- "command": ["sysctl", "-w", "vm.max_map_count=262144"],
- "securityContext": {
- "privileged": true
- }
- }
- ]'
- spec:
- containers:
- - name: es-master
- securityContext:
- privileged: false
- capabilities:
- add:
-# IPC_LOCK capability is enabled to allow Elasticsearch to lock the heap in memory so it will not be swapped.
- - IPC_LOCK
-# SYS_RESOURCE is docker capability key to control and override the resource limits.
-# This could be needed to increase base limits.(e.g. File descriptor limit for elasticsearch)
- - SYS_RESOURCE
- image: quay.io/pires/docker-elasticsearch-kubernetes:5.2.2
- env:
- - name: NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- - name: NODE_NAME
- valueFrom:
- fieldRef:
- fieldPath: metadata.name
- - name: "CLUSTER_NAME"
- value: "myesdb"
- - name: "NUMBER_OF_MASTERS"
- value: "2"
- - name: NODE_MASTER
- value: "true"
- - name: NODE_INGEST
- value: "false"
- - name: NODE_DATA
- value: "false"
- - name: HTTP_ENABLE
- value: "false"
- - name: "ES_JAVA_OPTS"
- value: "-Xms256m -Xmx256m"
- ports:
- - containerPort: 9300
- name: transport
- protocol: TCP
- volumeMounts:
- - name: storage
- mountPath: /data
- volumes:
- - emptyDir:
- medium: ""
- name: "storage"
----
-# Kubernetes deployment script for Elasticsearch client nodes (aka load balancing proxies).
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
- name: es-client
- labels:
- component: elasticsearch
- role: client
-spec:
- # The no. of replicas can be incremented based on the client usage using HTTP API.
- replicas: 1
- template:
- metadata:
- labels:
- component: elasticsearch
- role: client
- annotations:
- # Elasticsearch uses a hybrid mmapfs / niofs directory by default to store its indices.
- # The default operating system limits on mmap counts is likely to be too low, which may result
- # in out of memory exceptions. Therefore, the need to increase virtual memory
- # vm.max_map_count for large amount of data in the pod initialization annotation.
- pod.beta.kubernetes.io/init-containers: '[
- {
- "name": "sysctl",
- "image": "busybox",
- "imagePullPolicy": "IfNotPresent",
- "command": ["sysctl", "-w", "vm.max_map_count=262144"],
- "securityContext": {
- "privileged": true
- }
- }
- ]'
- spec:
- containers:
- - name: es-client
- securityContext:
- privileged: false
- capabilities:
- add:
-# IPC_LOCK capability is enabled to allow Elasticsearch to lock the heap in memory so it will not be swapped.
- - IPC_LOCK
-# SYS_RESOURCE is docker capability key to control and override the resource limits.
-# This could be needed to increase base limits.(e.g. File descriptor limit for elasticsearch)
- - SYS_RESOURCE
- image: quay.io/pires/docker-elasticsearch-kubernetes:5.2.2
- env:
- - name: NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- - name: NODE_NAME
- valueFrom:
- fieldRef:
- fieldPath: metadata.name
- - name: "CLUSTER_NAME"
- value: "myesdb"
- - name: NODE_MASTER
- value: "false"
- - name: NODE_DATA
- value: "false"
- - name: HTTP_ENABLE
- value: "true"
- - name: "ES_JAVA_OPTS"
- value: "-Xms256m -Xmx256m"
- ports:
- - containerPort: 9200
- name: http
- protocol: TCP
- - containerPort: 9300
- name: transport
- protocol: TCP
- volumeMounts:
- - name: storage
- mountPath: /data
- volumes:
- - emptyDir:
- medium: ""
- name: "storage"
----
-# Kubernetes deployment script for Elasticsearch data nodes which store and index data.
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
- name: es-data
- labels:
- component: elasticsearch
- role: data
-spec:
- replicas: 2
- template:
- metadata:
- labels:
- component: elasticsearch
- role: data
- annotations:
- pod.beta.kubernetes.io/init-containers: '[
- {
- "name": "sysctl",
- "image": "busybox",
- "imagePullPolicy": "IfNotPresent",
- "command": ["sysctl", "-w", "vm.max_map_count=1048575"],
- "securityContext": {
- "privileged": true
- }
- }
- ]'
- spec:
- containers:
- - name: es-data
- securityContext:
- privileged: false
- capabilities:
- add:
-# IPC_LOCK capability is enabled to allow Elasticsearch to lock the heap in memory so it will not be swapped.
- - IPC_LOCK
-# SYS_RESOURCE is docker capability key to control and override the resource limits.
-# This could be needed to increase base limits.(e.g. File descriptor limit for elasticsearch)
- - SYS_RESOURCE
- image: quay.io/pires/docker-elasticsearch-kubernetes:5.2.2
- env:
- - name: NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- - name: NODE_NAME
- valueFrom:
- fieldRef:
- fieldPath: metadata.name
- - name: "CLUSTER_NAME"
- value: "myesdb"
- - name: NODE_MASTER
- value: "false"
- - name: NODE_INGEST
- value: "false"
- - name: HTTP_ENABLE
- value: "false"
- - name: "ES_JAVA_OPTS"
- value: "-Xms256m -Xmx256m"
- ports:
- - containerPort: 9300
- name: transport
- protocol: TCP
- volumeMounts:
- - name: storage
- mountPath: /data
- volumes:
- - emptyDir:
- medium: ""
- name: "storage"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh b/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh
deleted file mode 100644
index 4d277c8..0000000
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/LargeProductionCluster/start-up.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#!/bin/sh
-set -e
-
-# Create Elasticsearch services and deployments.
-kubectl create -f es-services.yaml
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh b/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh
deleted file mode 100644
index a30793b..0000000
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/LargeProductionCluster/teardown.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/bin/bash
-set -e
-
-# Delete elasticsearch services and deployments.
-kubectl delete -f es-services.yaml
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/SmallITCluster/elasticsearch-svc-rc.yaml
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/SmallITCluster/elasticsearch-svc-rc.yaml b/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/SmallITCluster/elasticsearch-svc-rc.yaml
deleted file mode 100644
index 9a7ac3d..0000000
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/SmallITCluster/elasticsearch-svc-rc.yaml
+++ /dev/null
@@ -1,84 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# To create Elasticsearch frontend cluster Kubernetes service.
-# It sets up a load balancer on TCP port 9200 that distributes network traffic to the ES nodes.
-apiVersion: v1
-kind: Service
-metadata:
- name: elasticsearch
- labels:
- component: elasticsearch
-spec:
- type: LoadBalancer
- selector:
- component: elasticsearch
- ports:
- - name: http
- port: 9200
- protocol: TCP
- - name: transport
- port: 9300
- protocol: TCP
----
-# The Kubernetes deployment script for Elasticsearch replication nodes. It will create 1 node cluster.
-# To scale the cluster as desired, you can create replicas of node use 'kubectl scale --replicas=3 rc es' command
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
- name: es
- labels:
- component: elasticsearch
-spec:
- replicas: 1
- template:
- metadata:
- labels:
- component: elasticsearch
- spec:
- containers:
- - name: es
- securityContext:
- capabilities:
- add:
-# IPC_LOCK capability is enabled to allow Elasticsearch to lock the heap in memory so it will not be swapped.
- - IPC_LOCK
-# SYS_RESOURCE capability is set to control and override various resource limits.
- - SYS_RESOURCE
- image: quay.io/pires/docker-elasticsearch-kubernetes:5.2.2
- env:
- - name: "CLUSTER_NAME"
- value: "myesdb"
- - name: "DISCOVERY_SERVICE"
- value: "elasticsearch"
- - name: NODE_MASTER
- value: "true"
- - name: NODE_DATA
- value: "true"
- - name: HTTP_ENABLE
- value: "true"
- ports:
- - containerPort: 9200
- name: http
- protocol: TCP
- - containerPort: 9300
- name: transport
- protocol: TCP
- volumeMounts:
- - mountPath: /data
- name: storage
- volumes:
- - name: storage
- emptyDir: {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/SmallITCluster/start-up.sh
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/SmallITCluster/start-up.sh b/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/SmallITCluster/start-up.sh
deleted file mode 100644
index e8cf275..0000000
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/SmallITCluster/start-up.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#!/bin/sh
-set -e
-
-# Create Elasticsearch services and deployments.
-kubectl create -f elasticsearch-svc-rc.yaml
-
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/SmallITCluster/teardown.sh
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/SmallITCluster/teardown.sh b/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/SmallITCluster/teardown.sh
deleted file mode 100644
index 079141d..0000000
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/SmallITCluster/teardown.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/bin/bash
-set -e
-
-# Delete elasticsearch services and deployments.
-kubectl delete -f elasticsearch-svc-rc.yaml
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/data-load-setup.sh
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/data-load-setup.sh b/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/data-load-setup.sh
deleted file mode 100644
index 00991bc..0000000
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/data-load-setup.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/bin/bash
-set -e
-
-# Install python
-sudo apt-get update
-sudo apt-get install python-pip
-sudo pip install --upgrade pip
-sudo apt-get install python-dev
-sudo pip install tornado numpy
-echo
http://git-wip-us.apache.org/repos/asf/beam/blob/34b95076/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/data-load.sh
----------------------------------------------------------------------
diff --git a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/data-load.sh b/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/data-load.sh
deleted file mode 100644
index 21150fb..0000000
--- a/sdks/java/io/hadoop/jdk1.8-tests/src/test/resources/kubernetes/elasticsearch/data-load.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/bin/bash
-set -e
-
-# Identify external IP
-external_ip="$(kubectl get svc elasticsearch -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
-echo "Waiting for the Elasticsearch service to come up ........"
-while [ -z "$external_ip" ]
-do
- sleep 10s
- external_ip="$(kubectl get svc elasticsearch -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
- echo "."
-done
-echo "External IP - $external_ip"
-echo
-
-# Run the script
-/usr/bin/python es_test_data.py --count=1000 --format=Txn_ID:int,Item_Code:int,Item_ID:int,User_Name:str,last_updated:ts,Price:int,Title:str,Description:str,Age:int,Item_Name:str,Item_Price:int,Availability:bool,Batch_Num:int,Last_Ordered:tstxt,City:text --es_url=http://$external_ip:9200 &
\ No newline at end of file