You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2023/04/20 12:54:45 UTC
[beam] branch master updated: Update of GKE deployment (#25994)
This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 0be1dffdfe3 Update of GKE deployment (#25994)
0be1dffdfe3 is described below
commit 0be1dffdfe32bad2b2336170e9a6da783744ce64
Author: Rouslan <11...@users.noreply.github.com>
AuthorDate: Thu Apr 20 16:54:37 2023 +0400
Update of GKE deployment (#25994)
* Merge of PG deployment changes
* add cert manager api, doc fix
* DNS Administrator role added
* added prefix for firewall rules
* add licence to firewall.tf
* network name prefix for dns zones, allows multienv
* moved private access ips to locals. Added link to doc in comment
* fix issues
* AppEngineFlagChange
* Fules update
* Update readme file
---------
Co-authored-by: vdjerek <Vlado Djerek>
Co-authored-by: Sergey Makarkin <se...@akvelon.com>
---
playground/backend/containers/go/Dockerfile | 14 +-
playground/backend/containers/go/build.gradle | 8 -
playground/backend/containers/go/entrypoint.sh | 9 -
playground/backend/containers/java/Dockerfile | 17 +-
playground/backend/containers/java/build.gradle | 8 -
playground/backend/containers/java/entrypoint.sh | 9 -
playground/backend/containers/python/Dockerfile | 15 +-
playground/backend/containers/python/build.gradle | 8 -
playground/backend/containers/python/entrypoint.sh | 10 -
playground/backend/containers/scio/Dockerfile | 34 ++-
playground/backend/containers/scio/build.gradle | 8 -
playground/backend/containers/scio/entrypoint.sh | 9 -
playground/backend/new_scio_project.sh | 10 +-
.../templates/autoscaling-frontend.yaml | 6 +-
.../helm-playground/templates/autoscaling-go.yaml | 20 +-
.../templates/autoscaling-java.yaml | 14 +-
.../templates/autoscaling-python.yaml | 20 +-
.../templates/autoscaling-router.yaml | 6 +-
.../templates/autoscaling-scio.yaml | 14 +-
.../helm-playground/templates/backendconf.yml | 1 +
.../templates/deployment-frontend.yml | 2 +-
.../helm-playground/templates/deployment-go.yml | 10 +-
.../helm-playground/templates/deployment-java.yml | 12 +-
.../templates/deployment-python.yml | 8 +-
.../templates/deployment-router-grpc.yml | 2 +
.../templates/deployment-router.yml | 4 +-
.../helm-playground/templates/deployment-scio.yml | 12 +-
.../infrastructure/helm-playground/values.yaml | 15 +-
playground/infrastructure/proxy/allow_list.py | 33 ---
.../infrastructure/proxy/allow_list_proxy.py | 32 ---
playground/terraform/README.md | 69 +++---
.../applications/backend/backend-go/main.tf | 73 ------
.../applications/backend/backend-go/output.tf | 23 --
.../applications/backend/backend-go/variables.tf | 93 --------
.../applications/backend/backend-java/main.tf | 74 -------
.../applications/backend/backend-java/variables.tf | 99 ---------
.../applications/backend/backend-python/main.tf | 73 ------
.../backend/backend-python/variables.tf | 92 --------
.../applications/backend/backend-router/main.tf | 73 ------
.../applications/backend/backend-router/output.tf | 23 --
.../backend/backend-router/variables.tf | 93 --------
.../applications/backend/backend-scio/main.tf | 73 ------
.../applications/backend/backend-scio/output.tf | 23 --
.../applications/backend/backend-scio/variables.tf | 93 --------
playground/terraform/applications/backend/main.tf | 118 ----------
.../terraform/applications/backend/output.tf | 38 ----
.../terraform/applications/backend/variables.tf | 227 -------------------
playground/terraform/applications/default/main.tf | 69 ------
.../applications/frontend/dispatch/README.md | 35 ---
.../applications/frontend/dispatch/dispatch.yaml | 35 ---
playground/terraform/applications/frontend/main.tf | 50 -----
.../terraform/applications/frontend/output.tf | 24 --
.../terraform/applications/frontend/variables.tf | 55 -----
playground/terraform/applications/main.tf | 80 -------
playground/terraform/applications/output.tf | 42 ----
playground/terraform/applications/variables.tf | 244 ---------------------
playground/terraform/build.gradle.kts | 115 +---------
.../infrastructure/api_enable/variables.tf | 2 +-
.../terraform/infrastructure/appengine/main.tf | 5 +-
.../infrastructure/appengine/variables.tf | 10 +
.../terraform/infrastructure/firewall/firewall.tf | 85 +++++++
.../infrastructure/firewall/locals.tf} | 14 +-
.../firewall/variables.tf} | 12 +-
playground/terraform/infrastructure/gke/main.tf | 7 +-
playground/terraform/infrastructure/gke/output.tf | 4 +
.../terraform/infrastructure/gke/variables.tf | 9 +-
.../terraform/infrastructure/ip_address/main.tf | 4 +-
.../infrastructure/ip_address/variables.tf | 5 +-
playground/terraform/infrastructure/main.tf | 31 ++-
.../infrastructure/memorystore/variables.tf | 4 +-
.../infrastructure/private_dns/googleapis_com.tf | 52 +++++
.../private_dns/locals.tf} | 9 +-
.../private_dns}/variables.tf | 22 +-
.../terraform/infrastructure/private_dns/zones.tf | 63 ++++++
playground/terraform/infrastructure/variables.tf | 22 +-
playground/terraform/main.tf | 52 +----
playground/terraform/variables.tf | 216 +++---------------
settings.gradle.kts | 2 +-
78 files changed, 533 insertions(+), 2573 deletions(-)
diff --git a/playground/backend/containers/go/Dockerfile b/playground/backend/containers/go/Dockerfile
index 36e9ac8577b..3d218faa334 100644
--- a/playground/backend/containers/go/Dockerfile
+++ b/playground/backend/containers/go/Dockerfile
@@ -61,17 +61,6 @@ ENV PROPERTY_PATH=/opt/playground/backend/properties.yaml
## Copy build result
COPY src/configs /opt/playground/backend/configs/
-# Install mitmpoxy
-RUN mkdir /opt/mitmproxy &&\
- cd /opt/mitmproxy &&\
- wget -q https://snapshots.mitmproxy.org/7.0.4/mitmproxy-7.0.4-linux.tar.gz &&\
- tar -zxvf mitmproxy-7.0.4-linux.tar.gz &&\
- mkdir /usr/local/share/ca-certificates/extra
-COPY allow_list_proxy.py /opt/mitmproxy/
-COPY allow_list.py /opt/mitmproxy/
-ENV HTTP_PROXY="http://127.0.0.1:8081"
-ENV HTTPS_PROXY="http://127.0.0.1:8081"
-
COPY src/properties.yaml /opt/playground/backend/properties.yaml
COPY entrypoint.sh /
@@ -89,8 +78,7 @@ RUN groupadd --gid 20000 appgroup \
RUN mkdir -p /opt/playground/backend/executable_files/
# Chown all required files to the `appuser`.
-RUN chown -R appuser:appgroup /opt/playground/backend/executable_files/ \
- && chmod -R 777 /usr/local/share/ca-certificates/extra/ && chmod -R 777 /etc/ssl/certs && chmod +x /entrypoint.sh
+RUN chown -R appuser:appgroup /opt/playground/backend/executable_files/ && chmod +x /entrypoint.sh
# Switch to appuser
USER appuser
diff --git a/playground/backend/containers/go/build.gradle b/playground/backend/containers/go/build.gradle
index 0f89d056f3e..079d6f7aa4e 100644
--- a/playground/backend/containers/go/build.gradle
+++ b/playground/backend/containers/go/build.gradle
@@ -56,14 +56,6 @@ task copyDockerfileDependencies(type: Copy) {
from 'setup_sdk.sh'
into 'build/'
}
- copy {
- from '../../../infrastructure/proxy/allow_list.py'
- into 'build/'
- }
- copy {
- from '../../../infrastructure/proxy/allow_list_proxy.py'
- into 'build/'
- }
copy {
from '../../../playground'
into 'build/playground'
diff --git a/playground/backend/containers/go/entrypoint.sh b/playground/backend/containers/go/entrypoint.sh
index 63d77a6fef0..988302c2a71 100755
--- a/playground/backend/containers/go/entrypoint.sh
+++ b/playground/backend/containers/go/entrypoint.sh
@@ -14,13 +14,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-nohup /opt/mitmproxy/mitmdump -s /opt/mitmproxy/allow_list_proxy.py -p 8081 &
-while [ ! -f /home/appuser/.mitmproxy/mitmproxy-ca.pem ] ;
-do
- sleep 2
-done
-openssl x509 -in /home/appuser/.mitmproxy/mitmproxy-ca.pem -inform PEM -out /home/appuser/.mitmproxy/mitmproxy-ca.crt
-cp /home/appuser/.mitmproxy/mitmproxy-ca.crt /usr/local/share/ca-certificates/extra/
-update-ca-certificates
-
/opt/playground/backend/server_go_backend
diff --git a/playground/backend/containers/java/Dockerfile b/playground/backend/containers/java/Dockerfile
index 43f0d2e79e7..ae9a56d9c31 100644
--- a/playground/backend/containers/java/Dockerfile
+++ b/playground/backend/containers/java/Dockerfile
@@ -94,17 +94,6 @@ RUN cd /go/src/katas &&\
./gradlew jar &&\
mv util/build/libs/util.jar /opt/apache/beam/jars/util.jar
-# Install mitmpoxy
-RUN mkdir /opt/mitmproxy &&\
- cd /opt/mitmproxy &&\
- wget https://snapshots.mitmproxy.org/7.0.4/mitmproxy-7.0.4-linux.tar.gz &&\
- tar -zxvf mitmproxy-7.0.4-linux.tar.gz &&\
- mkdir /usr/local/share/ca-certificates/extra
-COPY allow_list_proxy.py /opt/mitmproxy/
-COPY allow_list.py /opt/mitmproxy/
-ENV HTTP_PROXY="http://127.0.0.1:8081"
-ENV HTTPS_PROXY="http://127.0.0.1:8081"
-
COPY src/properties.yaml /opt/playground/backend/properties.yaml
COPY entrypoint.sh /
COPY kafka-emulator/kafka-emulator.tar /opt/playground/backend/kafka-emulator/
@@ -119,9 +108,9 @@ RUN groupadd --gid 20000 appgroup \
RUN mkdir -p /opt/playground/backend/executable_files/
# Chown all required files to the `appuser`.
-RUN chown -R appuser:appgroup /opt/playground/backend/executable_files/ && chmod -R 775 /opt/apache/beam/jars/ \
- && chmod -R 777 /usr/local/share/ca-certificates/extra/ && chmod -R 777 /usr/local/openjdk-11/lib/security/ \
- && chmod -R 777 /etc/ssl/certs && chmod +x /entrypoint.sh
+
+RUN chown -R appuser:appgroup /opt/playground/backend/executable_files/ && chmod o+r /opt/apache/beam/jars/* \
+ && chmod +x /entrypoint.sh
# Switch to appuser
USER appuser
diff --git a/playground/backend/containers/java/build.gradle b/playground/backend/containers/java/build.gradle
index de6062e4ecc..5b236a9d1b4 100644
--- a/playground/backend/containers/java/build.gradle
+++ b/playground/backend/containers/java/build.gradle
@@ -53,14 +53,6 @@ task copyDockerfileDependencies(type: Copy) {
from 'entrypoint.sh'
into 'build/'
}
- copy {
- from '../../../infrastructure/proxy/allow_list.py'
- into 'build/'
- }
- copy {
- from '../../../infrastructure/proxy/allow_list_proxy.py'
- into 'build/'
- }
copy {
from '../../../playground'
into 'build/playground'
diff --git a/playground/backend/containers/java/entrypoint.sh b/playground/backend/containers/java/entrypoint.sh
index 87305f61055..146281d0640 100755
--- a/playground/backend/containers/java/entrypoint.sh
+++ b/playground/backend/containers/java/entrypoint.sh
@@ -14,13 +14,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-nohup /opt/mitmproxy/mitmdump -s /opt/mitmproxy/allow_list_proxy.py -p 8081 &
-while [ ! -f /home/appuser/.mitmproxy/mitmproxy-ca.pem ] ;
-do
- sleep 2
-done
-openssl x509 -in /home/appuser/.mitmproxy/mitmproxy-ca.pem -inform PEM -out /home/appuser/.mitmproxy/mitmproxy-ca.crt
-cp /home/appuser/.mitmproxy/mitmproxy-ca.crt /usr/local/share/ca-certificates/extra/
-update-ca-certificates
-
/opt/playground/backend/server_java_backend
diff --git a/playground/backend/containers/python/Dockerfile b/playground/backend/containers/python/Dockerfile
index f611b7b1acd..59454b6e97c 100644
--- a/playground/backend/containers/python/Dockerfile
+++ b/playground/backend/containers/python/Dockerfile
@@ -56,17 +56,6 @@ COPY --from=build /go/src/playground/backend/configs /opt/playground/backend/con
# Install Python Katas Utils
ENV PYTHONPATH="$PYTHONPATH:/go/src/katas"
-# Install mitmpoxy
-RUN mkdir /opt/mitmproxy &&\
- cd /opt/mitmproxy &&\
- wget https://snapshots.mitmproxy.org/7.0.4/mitmproxy-7.0.4-linux.tar.gz &&\
- tar -zxvf mitmproxy-7.0.4-linux.tar.gz &&\
- mkdir /usr/local/share/ca-certificates/extra
-COPY allow_list_proxy.py /opt/mitmproxy/
-COPY allow_list.py /opt/mitmproxy/
-ENV HTTP_PROXY="http://127.0.0.1:8081"
-ENV HTTPS_PROXY="http://127.0.0.1:8081"
-
COPY src/properties.yaml /opt/playground/backend/properties.yaml
COPY entrypoint.sh /
@@ -84,9 +73,7 @@ RUN groupadd --gid 20000 appgroup \
RUN mkdir -p /opt/playground/backend/executable_files/
# Chown all required files to the `appuser`.
-RUN chown -R appuser:appgroup /opt/playground/backend/executable_files/ \
- && chmod 777 /usr/local/lib/python3.10/site-packages/certifi/cacert.pem \
- && chmod -R 777 /usr/local/share/ca-certificates/extra/ && chmod -R 777 /etc/ssl/certs && chmod +x /entrypoint.sh
+RUN chown -R appuser:appgroup /opt/playground/backend/executable_files/ && chmod +x /entrypoint.sh
# Switch to appuser
USER appuser
diff --git a/playground/backend/containers/python/build.gradle b/playground/backend/containers/python/build.gradle
index dd665a9645b..95e1162c828 100644
--- a/playground/backend/containers/python/build.gradle
+++ b/playground/backend/containers/python/build.gradle
@@ -52,14 +52,6 @@ task copyDockerfileDependencies(type: Copy) {
from 'entrypoint.sh'
into 'build/'
}
- copy {
- from '../../../infrastructure/proxy/allow_list.py'
- into 'build/'
- }
- copy {
- from '../../../infrastructure/proxy/allow_list_proxy.py'
- into 'build/'
- }
copy {
from '../../../playground'
into 'build/playground'
diff --git a/playground/backend/containers/python/entrypoint.sh b/playground/backend/containers/python/entrypoint.sh
index b25293f1486..b8cfeb467cb 100755
--- a/playground/backend/containers/python/entrypoint.sh
+++ b/playground/backend/containers/python/entrypoint.sh
@@ -14,14 +14,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-nohup /opt/mitmproxy/mitmdump -s /opt/mitmproxy/allow_list_proxy.py -p 8081 &
-while [ ! -f /home/appuser/.mitmproxy/mitmproxy-ca.pem ] ;
-do
- sleep 2
-done
-openssl x509 -in /home/appuser/.mitmproxy/mitmproxy-ca.pem -inform PEM -out /home/appuser/.mitmproxy/mitmproxy-ca.crt
-cp /home/appuser/.mitmproxy/mitmproxy-ca.crt /usr/local/share/ca-certificates/extra/
-update-ca-certificates
-cat /home/appuser/.mitmproxy/mitmproxy-ca.pem >> /usr/local/lib/python3.10/site-packages/certifi/cacert.pem
-
/opt/playground/backend/server_python_backend
diff --git a/playground/backend/containers/scio/Dockerfile b/playground/backend/containers/scio/Dockerfile
index 4bb909ec636..13bf5f8c494 100644
--- a/playground/backend/containers/scio/Dockerfile
+++ b/playground/backend/containers/scio/Dockerfile
@@ -58,16 +58,6 @@ echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee /etc/apt/sources.l
curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | apt-key add
RUN apt-get update && apt-get install -y sbt
-
-## Install mitmpoxy
-RUN mkdir /opt/mitmproxy &&\
- cd /opt/mitmproxy &&\
- wget https://snapshots.mitmproxy.org/7.0.4/mitmproxy-7.0.4-linux.tar.gz &&\
- tar -zxvf mitmproxy-7.0.4-linux.tar.gz &&\
- mkdir /usr/local/share/ca-certificates/extra
-COPY allow_list_proxy.py /opt/mitmproxy/
-COPY allow_list.py /opt/mitmproxy/
-
COPY src/properties.yaml /opt/playground/backend/properties.yaml
COPY entrypoint.sh /
@@ -84,25 +74,27 @@ RUN groupadd --gid 20000 appgroup \
RUN mkdir -p /opt/playground/backend/executable_files/
# Chown all required files to the `appuser`.
-RUN chown -R appuser:appgroup /opt/playground/backend/executable_files/ \
- && chmod -R 777 /usr/local/share/ca-certificates/extra/ && chmod -R 777 /usr/local/openjdk-8/jre/lib/security/ \
- && chmod -R 777 /etc/ssl/certs && chmod +x /entrypoint.sh
+RUN chown -R appuser:appgroup /opt/playground/backend/executable_files/ && chmod +x /entrypoint.sh
+
+# Let sbt download files from Maven
+RUN mkdir -p /opt/sbt-template
+RUN chown -R appuser:appgroup /opt/sbt-template
# Switch to appuser
USER appuser
-# Let sbt download files from Maven
-RUN mkdir -p /tmp/sbt-initialize
-WORKDIR /tmp/sbt-initialize
+# Cache sbt project template
+WORKDIR /opt/sbt-template
RUN /opt/playground/backend/new_scio_project.sh
-WORKDIR /tmp/sbt-initialize/scio
+WORKDIR /opt/sbt-template/scio
RUN sbt "+compile"
WORKDIR /
-RUN rm -r /tmp/sbt-initialize
-# Enable mitmproxy
-ENV HTTP_PROXY="http://127.0.0.1:8081"
-ENV HTTPS_PROXY="http://127.0.0.1:8081"
+USER root
+RUN chown -R root:root /opt/sbt-template
+
+USER appuser
+
ENV SBT_OPTS="-Xmx512M -XX:+UseG1GC -XX:+UseStringDeduplication"
ENTRYPOINT ["/entrypoint.sh"]
diff --git a/playground/backend/containers/scio/build.gradle b/playground/backend/containers/scio/build.gradle
index 0050849a14f..145642b4a76 100644
--- a/playground/backend/containers/scio/build.gradle
+++ b/playground/backend/containers/scio/build.gradle
@@ -52,14 +52,6 @@ task copyDockerfileDependencies(type: Copy) {
from 'entrypoint.sh'
into 'build/'
}
- copy {
- from '../../../infrastructure/proxy/allow_list.py'
- into 'build/'
- }
- copy {
- from '../../../infrastructure/proxy/allow_list_proxy.py'
- into 'build/'
- }
copy {
from '../../../playground'
into 'build/playground'
diff --git a/playground/backend/containers/scio/entrypoint.sh b/playground/backend/containers/scio/entrypoint.sh
index b3e6d55ae50..fd2ef3d618e 100755
--- a/playground/backend/containers/scio/entrypoint.sh
+++ b/playground/backend/containers/scio/entrypoint.sh
@@ -14,13 +14,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-nohup /opt/mitmproxy/mitmdump -s /opt/mitmproxy/allow_list_proxy.py -p 8081 &
-while [ ! -f /home/appuser/.mitmproxy/mitmproxy-ca.pem ] ;
-do
- sleep 2
-done
-openssl x509 -in /home/appuser/.mitmproxy/mitmproxy-ca.pem -inform PEM -out /home/appuser/.mitmproxy/mitmproxy-ca.crt
-cp /home/appuser/.mitmproxy/mitmproxy-ca.crt /usr/local/share/ca-certificates/extra/
-update-ca-certificates
-
/opt/playground/backend/server_scio_backend
diff --git a/playground/backend/new_scio_project.sh b/playground/backend/new_scio_project.sh
index 80e031f0fd8..8ca358a7011 100755
--- a/playground/backend/new_scio_project.sh
+++ b/playground/backend/new_scio_project.sh
@@ -15,6 +15,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-{ printf scio\\nscio\\n; yes; } | sbt new spotify/scio-template.g8
+if [ -d /opt/sbt-template/scio ]; then
+ cp -r /opt/sbt-template/scio scio
+else
+ { printf scio\\nscio\\n; yes; } | sbt new spotify/scio-template.g8
+
+ echo "Compile / run / fork := false" >> scio/build.sbt
+ echo "updateOptions := updateOptions.value.withCachedResolution(true)" >> scio/build.sbt
+fi
-echo "Compile / run / fork := false" >> scio/build.sbt
diff --git a/playground/infrastructure/helm-playground/templates/autoscaling-frontend.yaml b/playground/infrastructure/helm-playground/templates/autoscaling-frontend.yaml
index 6fc40394330..b7f747da32e 100644
--- a/playground/infrastructure/helm-playground/templates/autoscaling-frontend.yaml
+++ b/playground/infrastructure/helm-playground/templates/autoscaling-frontend.yaml
@@ -17,10 +17,10 @@ kind: HorizontalPodAutoscaler
metadata:
name: playground-frontend
spec:
- maxReplicas: 10
- minReplicas: 1
+ maxReplicas: {{ .Values.autoscaling.rest.maxReplicas }}
+ minReplicas: {{ .Values.autoscaling.rest.minReplicas }}
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: playground-frontend
- targetCPUUtilizationPercentage: 90
+ targetCPUUtilizationPercentage: {{ .Values.autoscaling.utilization.cpuUtilization }}
diff --git a/playground/infrastructure/helm-playground/templates/autoscaling-go.yaml b/playground/infrastructure/helm-playground/templates/autoscaling-go.yaml
index 81a28e2d59a..b46f0165677 100644
--- a/playground/infrastructure/helm-playground/templates/autoscaling-go.yaml
+++ b/playground/infrastructure/helm-playground/templates/autoscaling-go.yaml
@@ -12,15 +12,27 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-apiVersion: autoscaling/v1
+apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: playground-go
spec:
- maxReplicas: 10
- minReplicas: 1
+ maxReplicas: {{ .Values.autoscaling.runners.maxReplicas }}
+ minReplicas: {{ .Values.autoscaling.runners.minReplicas }}
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: playground-go
- targetCPUUtilizationPercentage: 90
+ metrics:
+ - type: Resource
+ resource:
+ name: cpu
+ target:
+ type: Utilization
+ averageUtilization: {{ .Values.autoscaling.utilization.cpuUtilization }}
+ - type: Resource
+ resource:
+ name: memory
+ target:
+ type: Utilization
+ averageUtilization: {{ .Values.autoscaling.utilization.memoryUtilization }}
diff --git a/playground/infrastructure/helm-playground/templates/autoscaling-java.yaml b/playground/infrastructure/helm-playground/templates/autoscaling-java.yaml
index 7784c0432eb..9dc4991c0b6 100644
--- a/playground/infrastructure/helm-playground/templates/autoscaling-java.yaml
+++ b/playground/infrastructure/helm-playground/templates/autoscaling-java.yaml
@@ -12,15 +12,21 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-apiVersion: autoscaling/v1
+apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: playground-java
spec:
- maxReplicas: 10
- minReplicas: 2
+ maxReplicas: {{ .Values.autoscaling.runners.maxReplicas }}
+ minReplicas: {{ .Values.autoscaling.runners.minReplicas }}
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: playground-java
- targetCPUUtilizationPercentage: 90
+ metrics:
+ - type: Resource
+ resource:
+ name: memory
+ target:
+ type: Utilization
+ averageUtilization: {{ .Values.autoscaling.utilization.memoryUtilization }}
diff --git a/playground/infrastructure/helm-playground/templates/autoscaling-python.yaml b/playground/infrastructure/helm-playground/templates/autoscaling-python.yaml
index a4d96a7d035..d52cc6d856f 100644
--- a/playground/infrastructure/helm-playground/templates/autoscaling-python.yaml
+++ b/playground/infrastructure/helm-playground/templates/autoscaling-python.yaml
@@ -12,15 +12,27 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-apiVersion: autoscaling/v1
+apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: playground-python
spec:
- maxReplicas: 10
- minReplicas: 1
+ maxReplicas: {{ .Values.autoscaling.runners.maxReplicas }}
+ minReplicas: {{ .Values.autoscaling.runners.minReplicas }}
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: playground-python
- targetCPUUtilizationPercentage: 90
+ metrics:
+ - type: Resource
+ resource:
+ name: cpu
+ target:
+ type: Utilization
+ averageUtilization: {{ .Values.autoscaling.utilization.cpuUtilization }}
+ - type: Resource
+ resource:
+ name: memory
+ target:
+ type: Utilization
+ averageUtilization: {{ .Values.autoscaling.utilization.memoryUtilization }}
diff --git a/playground/infrastructure/helm-playground/templates/autoscaling-router.yaml b/playground/infrastructure/helm-playground/templates/autoscaling-router.yaml
index b13edbdf8af..7547b745bd2 100644
--- a/playground/infrastructure/helm-playground/templates/autoscaling-router.yaml
+++ b/playground/infrastructure/helm-playground/templates/autoscaling-router.yaml
@@ -17,10 +17,10 @@ kind: HorizontalPodAutoscaler
metadata:
name: playground-router
spec:
- maxReplicas: 10
- minReplicas: 1
+ maxReplicas: {{ .Values.autoscaling.rest.maxReplicas }}
+ minReplicas: {{ .Values.autoscaling.rest.minReplicas }}
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: playground-router
- targetCPUUtilizationPercentage: 90
+ targetCPUUtilizationPercentage: {{ .Values.autoscaling.utilization.cpuUtilization }}
diff --git a/playground/infrastructure/helm-playground/templates/autoscaling-scio.yaml b/playground/infrastructure/helm-playground/templates/autoscaling-scio.yaml
index c62f09bb8a1..8dd077774cd 100644
--- a/playground/infrastructure/helm-playground/templates/autoscaling-scio.yaml
+++ b/playground/infrastructure/helm-playground/templates/autoscaling-scio.yaml
@@ -12,15 +12,21 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-apiVersion: autoscaling/v1
+apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: playground-scio
spec:
- maxReplicas: 10
- minReplicas: 2
+ maxReplicas: {{ .Values.autoscaling.runners.maxReplicas }}
+ minReplicas: {{ .Values.autoscaling.runners.minReplicas }}
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: playground-scio
- targetCPUUtilizationPercentage: 90
+ metrics:
+ - type: Resource
+ resource:
+ name: memory
+ target:
+ type: Utilization
+ averageUtilization: {{ .Values.autoscaling.utilization.memoryUtilization }}
diff --git a/playground/infrastructure/helm-playground/templates/backendconf.yml b/playground/infrastructure/helm-playground/templates/backendconf.yml
index 725a9e13b13..fad0412754e 100644
--- a/playground/infrastructure/helm-playground/templates/backendconf.yml
+++ b/playground/infrastructure/helm-playground/templates/backendconf.yml
@@ -22,3 +22,4 @@ spec:
healthCheck:
type: HTTP
requestPath: /liveness
+ timeoutSec: 300
diff --git a/playground/infrastructure/helm-playground/templates/deployment-frontend.yml b/playground/infrastructure/helm-playground/templates/deployment-frontend.yml
index fa0f4916e00..994c4941f68 100644
--- a/playground/infrastructure/helm-playground/templates/deployment-frontend.yml
+++ b/playground/infrastructure/helm-playground/templates/deployment-frontend.yml
@@ -51,7 +51,7 @@ spec:
memory: "50Mi"
cpu: "250m"
limits:
- cpu: "500"
+ cpu: "500m"
memory: "500Mi"
replicas: {{ .Values.replicaCount }}
selector:
diff --git a/playground/infrastructure/helm-playground/templates/deployment-go.yml b/playground/infrastructure/helm-playground/templates/deployment-go.yml
index a9c9b4ecd79..387b47d3530 100644
--- a/playground/infrastructure/helm-playground/templates/deployment-go.yml
+++ b/playground/infrastructure/helm-playground/templates/deployment-go.yml
@@ -40,7 +40,9 @@ spec:
- name: CACHE_ADDRESS
value: {{ .Values.redis_ip }}
- name: NUM_PARALLEL_JOBS
- value: "10"
+ value: "30"
+ - name: DATASTORE_NAMESPACE
+ value: {{ .Values.datastore_name }}
livenessProbe:
httpGet:
path: /liveness
@@ -56,10 +58,10 @@ spec:
resources:
requests:
memory: "1000Mi"
- cpu: "250m"
+ cpu: "1000m"
limits:
- cpu: "500"
- memory: "1000Mi"
+ cpu: "4000m"
+ memory: "16000Mi"
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
diff --git a/playground/infrastructure/helm-playground/templates/deployment-java.yml b/playground/infrastructure/helm-playground/templates/deployment-java.yml
index 4ee1c485f86..c80ce7c9e56 100644
--- a/playground/infrastructure/helm-playground/templates/deployment-java.yml
+++ b/playground/infrastructure/helm-playground/templates/deployment-java.yml
@@ -39,7 +39,9 @@ spec:
- name: CACHE_ADDRESS
value: {{ .Values.redis_ip }}
- name: NUM_PARALLEL_JOBS
- value: "5"
+ value: "6"
+ - name: DATASTORE_NAMESPACE
+ value: {{ .Values.datastore_name }}
livenessProbe:
httpGet:
path: /liveness
@@ -54,11 +56,11 @@ spec:
periodSeconds: {{ .Values.healthcheck.readPeriodSeconds }}
resources:
requests:
- memory: "1000Mi"
- cpu: "500m"
+ memory: "6000Mi"
+ cpu: "1000m"
limits:
- cpu: "1000"
- memory: "1500Mi"
+ cpu: "8000m"
+ memory: "16000Mi"
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
diff --git a/playground/infrastructure/helm-playground/templates/deployment-python.yml b/playground/infrastructure/helm-playground/templates/deployment-python.yml
index 85fdecfc6d0..d53355dc9db 100644
--- a/playground/infrastructure/helm-playground/templates/deployment-python.yml
+++ b/playground/infrastructure/helm-playground/templates/deployment-python.yml
@@ -40,6 +40,8 @@ spec:
value: {{ .Values.redis_ip }}
- name: NUM_PARALLEL_JOBS
value: "20"
+ - name: DATASTORE_NAMESPACE
+ value: {{ .Values.datastore_name }}
livenessProbe:
httpGet:
path: /liveness
@@ -55,10 +57,10 @@ spec:
resources:
requests:
memory: "1000Mi"
- cpu: "250m"
+ cpu: "1000m"
limits:
- cpu: "500"
- memory: "1000Mi"
+ cpu: "4000m"
+ memory: "16000Mi"
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
diff --git a/playground/infrastructure/helm-playground/templates/deployment-router-grpc.yml b/playground/infrastructure/helm-playground/templates/deployment-router-grpc.yml
index abdd2b3bbce..b95bfb99c20 100644
--- a/playground/infrastructure/helm-playground/templates/deployment-router-grpc.yml
+++ b/playground/infrastructure/helm-playground/templates/deployment-router-grpc.yml
@@ -42,6 +42,8 @@ spec:
value: {{ .Values.redis_ip }}
- name: PROTOCOL_TYPE
value: TCP
+ - name: DATASTORE_NAMESPACE
+ value: {{ .Values.datastore_name }}
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
diff --git a/playground/infrastructure/helm-playground/templates/deployment-router.yml b/playground/infrastructure/helm-playground/templates/deployment-router.yml
index 710c2c3332c..7b2d20dca2c 100644
--- a/playground/infrastructure/helm-playground/templates/deployment-router.yml
+++ b/playground/infrastructure/helm-playground/templates/deployment-router.yml
@@ -42,6 +42,8 @@ spec:
value: {{ .Values.redis_ip }}
- name: NUM_PARALLEL_JOBS
value: "5"
+ - name: DATASTORE_NAMESPACE
+ value: {{ .Values.datastore_name }}
livenessProbe:
httpGet:
path: /liveness
@@ -59,7 +61,7 @@ spec:
memory: "50Mi"
cpu: "250m"
limits:
- cpu: "500"
+ cpu: "250m"
memory: "500Mi"
replicas: {{ .Values.replicaCount }}
selector:
diff --git a/playground/infrastructure/helm-playground/templates/deployment-scio.yml b/playground/infrastructure/helm-playground/templates/deployment-scio.yml
index 19421b014e8..f1b2a6d6cca 100644
--- a/playground/infrastructure/helm-playground/templates/deployment-scio.yml
+++ b/playground/infrastructure/helm-playground/templates/deployment-scio.yml
@@ -38,7 +38,9 @@ spec:
- name: CACHE_ADDRESS
value: {{ .Values.redis_ip }}
- name: NUM_PARALLEL_JOBS
- value: "5"
+ value: "6"
+ - name: DATASTORE_NAMESPACE
+ value: {{ .Values.datastore_name }}
livenessProbe:
httpGet:
path: /liveness
@@ -53,11 +55,11 @@ spec:
periodSeconds: {{ .Values.healthcheck.readPeriodSeconds }}
resources:
requests:
- memory: "1000Mi"
- cpu: "500m"
+ memory: "6000Mi"
+ cpu: "1000m"
limits:
- cpu: "1000"
- memory: "1500Mi"
+ cpu: "8000m"
+ memory: "16000Mi"
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
diff --git a/playground/infrastructure/helm-playground/values.yaml b/playground/infrastructure/helm-playground/values.yaml
index c051dd117f9..65ae5d15720 100644
--- a/playground/infrastructure/helm-playground/values.yaml
+++ b/playground/infrastructure/helm-playground/values.yaml
@@ -19,7 +19,7 @@ image:
router_image: beam_playground-backend-router
scio_image: beam_playground-backend-scio
python_image: beam_playground-backend-python
- frontend_image: beam_playground-frontend
+ frontend_image: beam_playground-frontend
pullPolicy: Always
service:
type: NodePort
@@ -30,4 +30,15 @@ healthcheck:
livInitialDelaySeconds: 30
livPeriodSeconds: 30
readInitialDelaySeconds: 30
- readPeriodSeconds: 30
\ No newline at end of file
+ readPeriodSeconds: 30
+autoscaling:
+ runners:
+ maxReplicas: 4
+ minReplicas: 2
+ rest:
+ maxReplicas: 4
+ minReplicas: 1
+ utilization:
+ memoryUtilization: 80
+ cpuUtilization: 95
+
diff --git a/playground/infrastructure/proxy/allow_list.py b/playground/infrastructure/proxy/allow_list.py
deleted file mode 100644
index aa7a9f2e08c..00000000000
--- a/playground/infrastructure/proxy/allow_list.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# ALLOWED_LIST contains all hosts that are allowed to make requests
-# from Beam Playground
-ALLOWED_LIST = [
- "localhost",
- "127.0.0.1",
- "logging.googleapis.com",
- "datastore.googleapis.com",
- "oauth2.googleapis.com",
- "storage.googleapis.com",
- "repo1.maven.org"
-]
-
-# ALLOWED_BUCKET_LIST contains all public Google Cloud Storage buckets
-# that are allowed to make requests from Beam Playground
-ALLOWED_BUCKET_LIST = [
- "dataflow-samples",
- "beam-samples"
-]
diff --git a/playground/infrastructure/proxy/allow_list_proxy.py b/playground/infrastructure/proxy/allow_list_proxy.py
deleted file mode 100644
index 5e71fea54d5..00000000000
--- a/playground/infrastructure/proxy/allow_list_proxy.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from mitmproxy import http
-
-from allow_list import ALLOWED_LIST, ALLOWED_BUCKET_LIST
-
-GCS_HOST = "storage.googleapis.com"
-
-
-def request(flow: http.HTTPFlow) -> None:
- allowed_bucket = flow.request.pretty_host == GCS_HOST and \
- (flow.request.path.split("/")[1] in ALLOWED_BUCKET_LIST or \
- flow.request.path.split("/")[4] in ALLOWED_BUCKET_LIST)
- allowed_host = flow.request.pretty_host in ALLOWED_LIST
- if not (allowed_bucket or allowed_host):
- flow.response = http.Response.make(
- status_code=403,
- content="Making requests to the hosts that are not listed "
- "in the allowed list is forbidden. "
- "host:" + flow.request.pretty_host + ", path: " + flow.request.path)
diff --git a/playground/terraform/README.md b/playground/terraform/README.md
index f472793fc34..4a3db70600c 100644
--- a/playground/terraform/README.md
+++ b/playground/terraform/README.md
@@ -30,20 +30,21 @@ Ensure that the account has at least following privileges:
- App Engine Admin
- App Engine Creator
- Artifact Registry Administrator
+ - Cloud Datastore Index Admin
- Cloud Memorystore Redis Admin
- Compute Admin
- Create Service Accounts
+ - DNS Administrator
- Kubernetes Engine Admin
- Quota Administrator
- Role Administrator
- Security Admin
- Service Account User
- Storage Admin
- - Cloud Datastore Index Admin
-3. [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for saving deployment state
+3. [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for Terraform state file
-4. DNS name for your Playground deployment instance
+4. DNS name for your Playground deployment instance further referred as <playground.zone>
5. OS with installed software listed below:
@@ -53,29 +54,37 @@ Ensure that the account has at least following privileges:
* [Docker](https://docs.docker.com/engine/install/)
* [Terraform](https://www.terraform.io/downloads)
* [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk)
+* [Kubectl authentication](https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke)
6. Apache Beam Git repository cloned locally
# Prepare deployment configuration:
-Playground uses `terraform.tfvars` located in `playground/terraform/environment/environment_name` to define variables specific to an environment (e.g., prod, test, staging).<br>
-1. Create a folder (further referred as `environment_name`) to define a new environment and place configuration files into it:
+Playground uses `terraform.tfvars` located in `playground/terraform/environment/<environment_name>` to define variables specific to an environment (e.g., prod, test, staging).<br>
+1. Create a folder (further referred as <environment_name>) to define a new environment and place configuration files into it:
* `terraform.tfvars` environment variables:
```
-project_id = "project_id" #GCP Project ID
-network_name = "network_name" #GCP VPC Network Name for Playground deployment
-gke_name = "playground-backend" #Playground GKE Cluster name
-region = "us-east1" #Set the deployment region
-location = "us-east1-b" #Select the deployment location from available in the specified region
-state_bucket = "bucket_name" #GCS bucket name for Beam Playground temp files
-redis_name = "playground_redis" #Choose the name for redis instance
-min_count = 2 #Min node count for GKE cluster
-max_count = 6 #Max node count for GKE cluster
+project_id = "project_id" # GCP Project ID
+network_name = "playground-network" # GCP VPC Network Name for Playground deployment
+subnetwork_name = "playground-subnetwork" # GCP VPC Subnetwork Name for Playground deployment
+gke_name = "playground-backend" # Playground GKE Cluster name
+region = "us-east1" # Deployment region for all created resources
+zone = "us-east1-b" # Deployment zone in the specified region
+state_bucket = "playground-state-bucket" # GCS bucket name for Terraform state file. The bucket must be created manually before deployment
+redis_name = "playground-redis" # Name for Redis instance
+redis_tier = "BASIC" # Redis tier type. Options: "Basic" or "Standard_HA". Default: "BASIC"
+min_count = 2 # Min node count for the GKE cluster
+max_count = 6 # Max node count for the GKE cluster
+skip_appengine_deploy = false # AppEngine flag - defined if AppEngine and Datastore need to be installed. Should be "true" if AppEngine and Datastore were installed before
+ip-address-name = "playground-static-ip" # GCP Static IP Address name
+repository_id = "playground-artifacts" # GCP Artifact repository name for Playground images
+service_account_id = "playground-gke-account" # GCP Service account name
+gke_machine_type = "e2-standard-8" # Machine type for GKE Nodes
```
* `state.tfbackend` environment variables:
```
-bucket = "bucket_name" #input bucket name - will be used for terraform tfstate file
+bucket = "playground-state-bucket" # GCS bucket name for Terraform state file. The same value as in terraform.tfvars file.
```
2. Configure authentication for the Google Cloud Platform
```
@@ -85,23 +94,23 @@ gcloud init
gcloud auth application-default login
```
# Deploy Playground infrastructure:
+If you intend to install more than one environment, you may need to manually remove 'playground/terraform/.terraform' folder before a new attempt
+
1. Start the following command from the top level repository folder ("beam") to deploy the Payground infrastructure:
```
-./gradlew playground:terraform:InitInfrastructure -Pproject_environment="environment_name" -Pdns-name="playground.zone"
+./gradlew playground:terraform:InitInfrastructure -Pproject_environment="<environment_name>"
```
-Where playground.zone - chosen DNS for Playground
2. Find a Static IP in your GCP project>VPC Network>IP Addresses>pg-static-ip
<br>Add following DNS A records for the discovered static IP address:
```
-java.playground.zone
-python.playground.zone
-scio.playground.zone
-go.playground.zone
-router.playground.zone
-playground.zone
+java.<playground.zone>
+python.<playground.zone>
+scio.<playground.zone>
+go.<playground.zone>
+router.<playground.zone>
+<playground.zone>
```
-Where "playground.zone" is the registered DNS zone<br>
[More about DNS zone registration](https://domains.google/get-started/domain-search/)<br>
[More about A records in DNS](https://support.google.com/a/answer/2579934?hl=en)
@@ -109,19 +118,19 @@ Where "playground.zone" is the registered DNS zone<br>
1. Run the following command to authenticate in the Docker registry:
```
-gcloud auth configure-docker <chosen_region>-docker.pkg.dev
+gcloud auth configure-docker <region>-docker.pkg.dev
```
2. Run the following command to authenticate in GKE:
```
-gcloud container clusters get-credentials --region <chosen_location> <gke_name> --project <project_id>
+gcloud container clusters get-credentials --region <zone> <gke_name> --project <project_id>
```
Start the following command from the top level repository folder ("beam") to deploy the Payground infrastructure:
```
-./gradlew playground:terraform:gkebackend -Pproject_environment="environment_name" -Pdocker-tag="tag" \
- -Pdns-name="playground.zone" -Psdk-tag=2.44.0 \
- -Pdocker-repository-root="<chosen_region>-docker.pkg.dev/<project_id>/playground-repository"
+./gradlew playground:terraform:gkebackend -Pproject_environment="<environment_name>" -Pdocker-tag="tag" \
+ -Pdns-name="<playground.zone>" -Psdk-tag=2.44.0 \
+ -Pdocker-repository-root="<region>-docker.pkg.dev/<project_id>/<repository_id>" -Pdatastore-namespace="Playground"
```
-Where tag - image tag for backend, playground.zone - chosen DNS for Playground, Psdk-tag - current BEAM version
+Where tag - image tag for docker images, Psdk-tag - current Apache Beam SDK version, Pdatastore-namespace - namespace for Datastore
During script execution, a Google managed certificate will be created. [Provisioning might take up to 60 minutes](https://cloud.google.com/load-balancing/docs/ssl-certificates/google-managed-certs).
diff --git a/playground/terraform/applications/backend/backend-go/main.tf b/playground/terraform/applications/backend/backend-go/main.tf
deleted file mode 100644
index a01764be4e3..00000000000
--- a/playground/terraform/applications/backend/backend-go/main.tf
+++ /dev/null
@@ -1,73 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-
-resource "google_app_engine_flexible_app_version" "backend_app_go" {
- version_id = "v1"
- project = var.project_id
- service = "${var.service_name}-${var.environment}"
- runtime = "custom"
- delete_service_on_destroy = true
-
- liveness_check {
- path = ""
- }
-
- readiness_check {
- path = "/readiness"
- }
-
- automatic_scaling {
- max_total_instances = var.max_instance
- min_total_instances = var.min_instance
- cool_down_period = "120s"
- cpu_utilization {
- target_utilization = 0.7
- }
- }
-
- resources {
- memory_gb = var.memory
- cpu = var.cpu
- volumes {
- name = "inmemory"
- size_gb = var.volume_size
- volume_type = "tmpfs"
- }
- }
-
- network {
- name = var.network_name
- subnetwork = var.subnetwork_name
- }
-
- env_variables = {
- CACHE_TYPE = var.cache_type
- CACHE_ADDRESS = "${var.cache_address}:6379"
- NUM_PARALLEL_JOBS = 30
- LAUNCH_SITE = "app_engine"
- }
-
- deployment {
- container {
- image = "${var.docker_registry_address}/${var.docker_image_name}:${var.docker_image_tag}"
- }
- }
-}
-
diff --git a/playground/terraform/applications/backend/backend-go/output.tf b/playground/terraform/applications/backend/backend-go/output.tf
deleted file mode 100644
index 2eb125a4136..00000000000
--- a/playground/terraform/applications/backend/backend-go/output.tf
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-output "server-url" {
- value = google_app_engine_flexible_app_version.backend_app_go.service
-}
-
diff --git a/playground/terraform/applications/backend/backend-go/variables.tf b/playground/terraform/applications/backend/backend-go/variables.tf
deleted file mode 100644
index 2d4aa78355f..00000000000
--- a/playground/terraform/applications/backend/backend-go/variables.tf
+++ /dev/null
@@ -1,93 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-variable "project_id" {
- description = "Project ID"
-}
-
-variable "docker_registry_address" {
- description = "Docker registry address"
-}
-
-variable "docker_image_name" {
- description = "Docker Image Name To Be Deployed"
- default = "beam_playground-backend-go"
-}
-
-variable "docker_image_tag" {
- description = "Docker Image Tag To Be Deployed"
- default = "latest"
-}
-
-variable "service_name" {
- default = "backend-go"
-}
-
-variable "cache_type" {
- default = "remote"
-}
-
-variable "cache_address" {
- default = ""
-}
-
-variable "network_name" {
- description = "VPC Name"
- default = "playground-vpc"
-}
-
-variable "subnetwork_name" {
- description = "Vpc Name"
- default = "playground-vpc"
-}
-
-variable "environment" {
- description = "prod,dev,beta"
- default = "dev"
-}
-
-variable "volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "min_instance" {
- description = "Min count instance app"
- type = number
- default = 2
-}
-
-variable "memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "cpu" {
- description = "CPU on instance, demo project quota 24 cpu on region (use 2 cpu) "
- type = number
- default = 8
-}
diff --git a/playground/terraform/applications/backend/backend-java/main.tf b/playground/terraform/applications/backend/backend-java/main.tf
deleted file mode 100644
index 94b093f4f91..00000000000
--- a/playground/terraform/applications/backend/backend-java/main.tf
+++ /dev/null
@@ -1,74 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-resource "google_app_engine_flexible_app_version" "backend_app_java" {
- version_id = "v1"
- project = var.project_id
- service = "${var.service_name}-${var.environment}"
- runtime = "custom"
- delete_service_on_destroy = true
-
-
- liveness_check {
- path = "/liveness"
- initial_delay = "40s"
- }
-
- readiness_check {
- path = "/readiness"
- }
-
- automatic_scaling {
- max_total_instances = var.max_instance
- min_total_instances = var.min_instance
- cool_down_period = "120s"
- cpu_utilization {
- target_utilization = 0.7
- }
- }
-
- env_variables = {
- CACHE_TYPE = var.cache_type
- CACHE_ADDRESS = "${var.cache_address}:6379"
- NUM_PARALLEL_JOBS = 10
- LAUNCH_SITE = "app_engine"
- }
-
- resources {
- memory_gb = var.memory
- cpu = var.cpu
- volumes {
- name = "inmemory"
- size_gb = var.volume_size
- volume_type = "tmpfs"
- }
- }
-
- network {
- name = var.network_name
- subnetwork = var.subnetwork_name
- }
-
- deployment {
- container {
- image = "${var.docker_registry_address}/${var.docker_image_name}:${var.docker_image_tag}"
- }
- }
-}
-
diff --git a/playground/terraform/applications/backend/backend-java/variables.tf b/playground/terraform/applications/backend/backend-java/variables.tf
deleted file mode 100644
index 1e44eb525f4..00000000000
--- a/playground/terraform/applications/backend/backend-java/variables.tf
+++ /dev/null
@@ -1,99 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-variable "project_id" {
- description = "Project ID"
-}
-
-variable "docker_registry_address" {
- description = "Docker registry address"
-}
-
-variable "docker_image_name" {
- description = "Docker Image Name To Be Deployed"
- default = "beam_playground-backend-java"
-}
-
-variable "docker_image_tag" {
- description = "Docker Image Tag To Be Deployed"
- default = "latest"
-}
-
-variable "memory_size" {
- description = "RAM in GB. The requested memory for the application"
- type = number
- default = 2
-}
-
-variable "volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "service_name" {
- default = "backend-java"
-}
-
-variable "cache_type" {
- default = "remote"
-}
-
-variable "cache_address" {
- default = ""
-}
-
-variable "network_name" {
- description = "VPC Name"
- default = "playground-vpc"
-}
-
-variable "subnetwork_name" {
- description = "Vpc Name"
- default = "playground-vpc"
-}
-
-variable "environment" {
- description = "prod,dev,beta"
- default = "dev"
-}
-
-variable "max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "min_instance" {
- description = "Min count instance app"
- type = number
- default = 2
-}
-
-variable "memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "cpu" {
- description = "CPU on instance, demo project quota 24 cpu on region (use 2 cpu)"
- type = number
- default = 8
-}
diff --git a/playground/terraform/applications/backend/backend-python/main.tf b/playground/terraform/applications/backend/backend-python/main.tf
deleted file mode 100644
index d579e87461d..00000000000
--- a/playground/terraform/applications/backend/backend-python/main.tf
+++ /dev/null
@@ -1,73 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-resource "google_app_engine_flexible_app_version" "backend_app_python" {
- version_id = "v1"
- project = var.project_id
- service = "${var.service_name}-${var.environment}"
- runtime = "custom"
- delete_service_on_destroy = true
-
- liveness_check {
- path = "/liveness"
- initial_delay = "40s"
- }
-
- readiness_check {
- path = "/readiness"
- }
-
- automatic_scaling {
- max_total_instances = var.max_instance
- min_total_instances = var.min_instance
- cool_down_period = "120s"
- cpu_utilization {
- target_utilization = 0.7
- }
- }
-
- network {
- name = var.network_name
- subnetwork = var.subnetwork_name
- }
-
- resources {
- memory_gb = var.memory
- cpu = var.cpu
- volumes {
- name = "inmemory"
- size_gb = var.volume_size
- volume_type = "tmpfs"
- }
- }
-
- env_variables = {
- CACHE_TYPE = var.cache_type
- CACHE_ADDRESS = "${var.cache_address}:6379"
- NUM_PARALLEL_JOBS = 70
- LAUNCH_SITE = "app_engine"
- }
-
- deployment {
- container {
- image = "${var.docker_registry_address}/${var.docker_image_name}:${var.docker_image_tag}"
- }
- }
-}
-
diff --git a/playground/terraform/applications/backend/backend-python/variables.tf b/playground/terraform/applications/backend/backend-python/variables.tf
deleted file mode 100644
index 273d67d783c..00000000000
--- a/playground/terraform/applications/backend/backend-python/variables.tf
+++ /dev/null
@@ -1,92 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-variable "project_id" {
- description = "Project ID"
-}
-
-variable "docker_registry_address" {
- description = "Docker registry address"
-}
-
-variable "docker_image_name" {
- description = "Docker Image Name To Be Deployed"
- default = "beam_playground-backend-python"
-}
-
-variable "docker_image_tag" {
- description = "Docker Image Tag To Be Deployed"
- default = "latest"
-}
-
-variable "service_name" {
- default = "backend-python"
-}
-variable "cache_type" {
- default = "remote"
-}
-
-variable "cache_address" {
- default = ""
-}
-
-variable "network_name" {
- description = "VPC Name"
- default = "playground-vpc"
-}
-
-variable "subnetwork_name" {
- description = "Vpc Name"
- default = "playground-vpc"
-}
-
-variable "environment" {
- description = "prod,dev,beta"
- default = "dev"
-}
-
-variable "volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "min_instance" {
- description = "Min count instance app"
- type = number
- default = 2
-}
-
-variable "memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "cpu" {
- description = "CPU on instance, demo project quota 24 cpu on region (use 2 cpu)"
- type = number
- default = 8
-}
\ No newline at end of file
diff --git a/playground/terraform/applications/backend/backend-router/main.tf b/playground/terraform/applications/backend/backend-router/main.tf
deleted file mode 100644
index 37a8eae19ab..00000000000
--- a/playground/terraform/applications/backend/backend-router/main.tf
+++ /dev/null
@@ -1,73 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-resource "google_app_engine_flexible_app_version" "backend_app_router" {
- version_id = "v1"
- project = var.project_id
- service = "${var.service_name}-${var.environment}"
- runtime = "custom"
- delete_service_on_destroy = true
-
- liveness_check {
- path = "/liveness"
- initial_delay = "40s"
- }
-
- readiness_check {
- path = "/readiness"
- }
-
- automatic_scaling {
- max_total_instances = var.max_instance
- min_total_instances = var.min_instance
- cool_down_period = "120s"
- cpu_utilization {
- target_utilization = 0.7
- }
- }
-
- resources {
- memory_gb = var.memory
- cpu = var.cpu
- volumes {
- name = "inmemory"
- size_gb = var.volume_size
- volume_type = "tmpfs"
- }
- }
-
- network {
- name = var.network_name
- subnetwork = var.subnetwork_name
- }
-
- env_variables = {
- CACHE_TYPE = var.cache_type
- CACHE_ADDRESS = "${var.cache_address}:6379"
- NUM_PARALLEL_JOBS = 10
- LAUNCH_SITE = "app_engine"
- }
-
- deployment {
- container {
- image = "${var.docker_registry_address}/${var.docker_image_name}:${var.docker_image_tag}"
- }
- }
-}
-
diff --git a/playground/terraform/applications/backend/backend-router/output.tf b/playground/terraform/applications/backend/backend-router/output.tf
deleted file mode 100644
index 32795607d53..00000000000
--- a/playground/terraform/applications/backend/backend-router/output.tf
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-output "server-url" {
- value = google_app_engine_flexible_app_version.backend_app_router.service
-}
-
diff --git a/playground/terraform/applications/backend/backend-router/variables.tf b/playground/terraform/applications/backend/backend-router/variables.tf
deleted file mode 100644
index 7fc6e74e883..00000000000
--- a/playground/terraform/applications/backend/backend-router/variables.tf
+++ /dev/null
@@ -1,93 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-variable "project_id" {
- description = "Project ID"
-}
-
-variable "docker_registry_address" {
- description = "Docker registry address"
-}
-
-variable "docker_image_name" {
- description = "Docker Image Name To Be Deployed"
- default = "beam_playground-backend-router"
-}
-
-variable "docker_image_tag" {
- description = "Docker Image Tag To Be Deployed"
- default = "latest"
-}
-
-variable "service_name" {
- default = "backend-router"
-}
-
-variable "cache_type" {
- default = "remote"
-}
-
-variable "cache_address" {
- default = ""
-}
-
-variable "network_name" {
- description = "VPC Name"
- default = "playground-vpc"
-}
-
-variable "subnetwork_name" {
- description = "Vpc Name"
- default = "playground-vpc"
-}
-
-variable "environment" {
- description = "prod,dev,beta"
- default = "dev"
-}
-
-variable "volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "max_instance" {
- description = "Max count instance app"
- type = number
- default = 3
-}
-
-variable "min_instance" {
- description = "Min count instance app"
- type = number
- default = 1
-}
-
-variable "memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 4
-}
-
-variable "cpu" {
- description = "CPU on instance, demo project quota 24 cpu on region (use 2 cpu)"
- type = number
- default = 2
-}
diff --git a/playground/terraform/applications/backend/backend-scio/main.tf b/playground/terraform/applications/backend/backend-scio/main.tf
deleted file mode 100644
index 84ad6302132..00000000000
--- a/playground/terraform/applications/backend/backend-scio/main.tf
+++ /dev/null
@@ -1,73 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-resource "google_app_engine_flexible_app_version" "backend_app_scio" {
- version_id = "v1"
- project = var.project_id
- service = "${var.service_name}-${var.environment}"
- runtime = "custom"
- delete_service_on_destroy = true
-
- liveness_check {
- path = "/liveness"
- initial_delay = "40s"
- }
-
- readiness_check {
- path = "/readiness"
- }
-
- automatic_scaling {
- max_total_instances = var.max_instance
- min_total_instances = var.min_instance
- cool_down_period = "120s"
- cpu_utilization {
- target_utilization = 0.7
- }
- }
-
- resources {
- memory_gb = var.memory
- cpu = var.cpu
- volumes {
- name = "inmemory"
- size_gb = var.volume_size
- volume_type = "tmpfs"
- }
- }
-
- network {
- name = var.network_name
- subnetwork = var.subnetwork_name
- }
-
- env_variables = {
- CACHE_TYPE = var.cache_type
- CACHE_ADDRESS = "${var.cache_address}:6379"
- NUM_PARALLEL_JOBS = 10
- LAUNCH_SITE = "app_engine"
- }
-
- deployment {
- container {
- image = "${var.docker_registry_address}/${var.docker_image_name}:${var.docker_image_tag}"
- }
- }
-}
-
diff --git a/playground/terraform/applications/backend/backend-scio/output.tf b/playground/terraform/applications/backend/backend-scio/output.tf
deleted file mode 100644
index e2575250527..00000000000
--- a/playground/terraform/applications/backend/backend-scio/output.tf
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-output "server-url" {
- value = google_app_engine_flexible_app_version.backend_app_scio.service
-}
-
diff --git a/playground/terraform/applications/backend/backend-scio/variables.tf b/playground/terraform/applications/backend/backend-scio/variables.tf
deleted file mode 100644
index 0fe4d4b2a4c..00000000000
--- a/playground/terraform/applications/backend/backend-scio/variables.tf
+++ /dev/null
@@ -1,93 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-variable "project_id" {
- description = "Project ID"
-}
-
-variable "docker_registry_address" {
- description = "Docker registry address"
-}
-
-variable "docker_image_name" {
- description = "Docker Image Name To Be Deployed"
- default = "beam_playground-backend-scio"
-}
-
-variable "docker_image_tag" {
- description = "Docker Image Tag To Be Deployed"
- default = "latest"
-}
-
-variable "service_name" {
- default = "backend-scio"
-}
-
-variable "cache_type" {
- default = "remote"
-}
-
-variable "cache_address" {
- default = ""
-}
-
-variable "volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "network_name" {
- description = "VPC Name"
- default = "playground-vpc"
-}
-
-variable "subnetwork_name" {
- description = "Vpc Name"
- default = "playground-vpc"
-}
-
-variable "environment" {
- description = "prod,dev,beta"
- default = "dev"
-}
-
-variable "max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "min_instance" {
- description = "Min count instance app"
- type = number
- default = 2
-}
-
-variable "memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "cpu" {
- description = "CPU on instance, demo project quota 24 cpu on region (use 2 cpu)"
- type = number
- default = 8
-}
diff --git a/playground/terraform/applications/backend/main.tf b/playground/terraform/applications/backend/main.tf
deleted file mode 100644
index 9c4ac961a53..00000000000
--- a/playground/terraform/applications/backend/main.tf
+++ /dev/null
@@ -1,118 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-
-
-module "backend-go" {
- source = "./backend-go"
- project_id = var.project_id
- cache_address = var.cache_address
- docker_registry_address = var.docker_registry_address
- network_name = var.network_name
- subnetwork_name = var.subnetwork_name
- environment = var.environment
- docker_image_name = "${var.docker_image_name}-go"
- docker_image_tag = var.docker_image_tag
- service_name = "${var.base_service_name}-go"
- cache_type = var.cache_type
- volume_size = var.go_volume_size
- cpu = var.go_cpu
- memory = var.go_memory
- max_instance = var.go_max_instance
- min_instance = var.go_min_instance
-}
-
-module "backend-java" {
- source = "./backend-java"
- project_id = var.project_id
- cache_address = var.cache_address
- docker_registry_address = var.docker_registry_address
- network_name = var.network_name
- subnetwork_name = var.subnetwork_name
- environment = var.environment
- docker_image_name = "${var.docker_image_name}-java"
- docker_image_tag = var.docker_image_tag
- service_name = "${var.base_service_name}-java"
- cache_type = var.cache_type
- volume_size = var.java_volume_size
- cpu = var.java_cpu
- memory = var.java_memory
- max_instance = var.java_max_instance
- min_instance = var.java_min_instance
-}
-
-module "backend-python" {
- source = "./backend-python"
- project_id = var.project_id
- cache_address = var.cache_address
- docker_registry_address = var.docker_registry_address
- network_name = var.network_name
- subnetwork_name = var.subnetwork_name
- environment = var.environment
- docker_image_name = "${var.docker_image_name}-python"
- docker_image_tag = var.docker_image_tag
- service_name = "${var.base_service_name}-python"
- cache_type = var.cache_type
- volume_size = var.python_volume_size
- cpu = var.python_cpu
- memory = var.python_memory
- max_instance = var.python_max_instance
- min_instance = var.python_min_instance
-}
-
-module "backend-router" {
- source = "./backend-router"
- project_id = var.project_id
- cache_address = var.cache_address
- docker_registry_address = var.docker_registry_address
- network_name = var.network_name
- subnetwork_name = var.subnetwork_name
- environment = var.environment
- docker_image_name = "${var.docker_image_name}-router"
- docker_image_tag = var.docker_image_tag
- service_name = "${var.base_service_name}-router"
- cache_type = var.cache_type
- volume_size = var.router_volume_size
- cpu = var.router_cpu
- memory = var.router_memory
- max_instance = var.router_max_instance
- min_instance = var.router_min_instance
-}
-
-module "backend-scio" {
- source = "./backend-scio"
- project_id = var.project_id
- cache_address = var.cache_address
- docker_registry_address = var.docker_registry_address
- network_name = var.network_name
- subnetwork_name = var.subnetwork_name
- environment = var.environment
- docker_image_name = "${var.docker_image_name}-scio"
- docker_image_tag = var.docker_image_tag
- service_name = "${var.base_service_name}-scio"
- cache_type = var.cache_type
- volume_size = var.scio_volume_size
- cpu = var.scio_cpu
- memory = var.scio_memory
- max_instance = var.scio_max_instance
- min_instance = var.scio_min_instance
-}
-
-
-
diff --git a/playground/terraform/applications/backend/output.tf b/playground/terraform/applications/backend/output.tf
deleted file mode 100644
index e18f755f2d6..00000000000
--- a/playground/terraform/applications/backend/output.tf
+++ /dev/null
@@ -1,38 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-output "go-server-url" {
- value = module.backend-go.server-url
-}
-
-output "java-server-url" {
- value = module.backend-java.server-url
-}
-
-output "python-server-url" {
- value = module.backend-python.server-url
-}
-
-output "router-server-url" {
- value = module.backend-router.server-url
-}
-
-output "scio-server-url" {
- value = module.backend-scio.server-url
-}
\ No newline at end of file
diff --git a/playground/terraform/applications/backend/variables.tf b/playground/terraform/applications/backend/variables.tf
deleted file mode 100644
index 913595e020c..00000000000
--- a/playground/terraform/applications/backend/variables.tf
+++ /dev/null
@@ -1,227 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-variable "project_id" {
- description = "The GCP Project ID where Playground Applications will be created"
-}
-
-variable "cache_address" {
- description = "IP Address for Redis service"
- default = ""
-}
-
-variable "environment" {
- description = "prod,dev,beta"
- default = "dev"
-}
-
-variable "network_name" {
- description = "VPC Name"
- default = "playground-vpc"
-}
-
-variable "subnetwork_name" {
- description = "Vpc Name"
- default = "playground-vpc"
-}
-
-variable "docker_image_tag" {
- description = "Docker Image Tag To Be Deployed"
- default = ""
-}
-
-variable "docker_image_name" {
- default = "beam_playground-backend"
- description = "Base prefix for docker images"
-}
-
-variable "docker_registry_address" {
- description = "Docker registry address"
-}
-
-variable "base_service_name" {
- default = "backend"
-}
-
-variable "cache_type" {
- description = "remote or local. Set remote to use Redis and local to use in-memory application level cache"
- default = "remote"
-}
-
-
-# SCIO variables
-
-variable "scio_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "scio_max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "scio_min_instance" {
- description = "Min count instance app"
- type = number
- default = 2
-}
-
-variable "scio_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "scio_cpu" {
- description = "CPU on instance, demo project quota 24 cpu on region (use 2 cpu)"
- type = number
- default = 8
-}
-
-# Go variables
-
-variable "go_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "go_max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "go_min_instance" {
- description = "Min count instance app"
- type = number
- default = 2
-}
-
-variable "go_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "go_cpu" {
- description = "CPU on instance, demo project quota 24 cpu on region (use 2 cpu)"
- type = number
- default = 8
-}
-
-# Python variables
-
-variable "python_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "python_max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "python_min_instance" {
- description = "Min count instance app"
- type = number
- default = 2
-}
-
-variable "python_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "python_cpu" {
- description = "CPU on instance, demo project quota 24 cpu on region (use 2 cpu)"
- type = number
- default = 8
-}
-
-# Java variables
-
-variable "java_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "java_max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "java_min_instance" {
- description = "Min count instance app"
- type = number
- default = 2
-}
-
-variable "java_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "java_cpu" {
- description = "CPU on instance, demo project quota 24 cpu on region (use 2 cpu)"
- type = number
- default = 8
-}
-
-# Router variables
-
-variable "router_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "router_max_instance" {
- description = "Max count instance app"
- type = number
- default = 3
-}
-
-variable "router_min_instance" {
- description = "Min count instance app"
- type = number
- default = 1
-}
-
-variable "router_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 4
-}
-
-variable "router_cpu" {
- description = "CPU on instance, demo project quota 24 cpu on region (use 2 cpu)"
- type = number
- default = 2
-}
-
diff --git a/playground/terraform/applications/default/main.tf b/playground/terraform/applications/default/main.tf
deleted file mode 100644
index ed9bf7edaf9..00000000000
--- a/playground/terraform/applications/default/main.tf
+++ /dev/null
@@ -1,69 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-resource "google_app_engine_application" "app_playground" {
- project = var.project_id
- location_id = var.location
- database_type = "CLOUD_DATASTORE_COMPATIBILITY"
-}
-
-resource "google_project_service" "firestore" {
- project = var.project_id
- service = "firestore.googleapis.com"
- disable_dependent_services = true
- depends_on = [
- google_app_engine_application.app_playground
- ]
-}
-
-resource "google_app_engine_flexible_app_version" "default_app" {
- count = var.create_default_service ? 1 : 0
- service = "default"
- version_id = "mlflow-default"
- runtime = "custom"
- project = var.project_id
-
- deployment {
- container {
- image = "gcr.io/cloudrun/hello"
- }
- }
-
- liveness_check {
- path = "/"
- }
-
- readiness_check {
- path = "/"
- }
-
- automatic_scaling {
- cool_down_period = "120s"
- min_total_instances = 1
- max_total_instances = 1
- cpu_utilization {
- target_utilization = 0.5
- }
- }
-
- delete_service_on_destroy = false
- noop_on_destroy = true
-
-}
-
diff --git a/playground/terraform/applications/frontend/dispatch/README.md b/playground/terraform/applications/frontend/dispatch/README.md
deleted file mode 100644
index bafa98ad9a2..00000000000
--- a/playground/terraform/applications/frontend/dispatch/README.md
+++ /dev/null
@@ -1,35 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-## Add Domain
-
-## Load dispatch to AppEngine
-
-Add mapping to external domain:
-
-```bash
-$ gcloud app deploy dispatch.yaml --project <project id>
-#
-```
-
-## Config frontend application
-
-
-
-
diff --git a/playground/terraform/applications/frontend/dispatch/dispatch.yaml b/playground/terraform/applications/frontend/dispatch/dispatch.yaml
deleted file mode 100644
index b51c98605ac..00000000000
--- a/playground/terraform/applications/frontend/dispatch/dispatch.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-dispatch:
- - url: "play.beam.apache.org/*"
- service: frontend-beta
- - url: "*/router/*"
- service: backend-router-beta
- - url: "*/scio/*"
- service: backend-scio-beta
- - url: "*/python/*"
- service: backend-python-beta
- - url: "*/go/*"
- service: backend-go-beta
- - url: "*/java/*"
- service: backend-java-beta
-
-
-
diff --git a/playground/terraform/applications/frontend/main.tf b/playground/terraform/applications/frontend/main.tf
deleted file mode 100644
index 53cea705297..00000000000
--- a/playground/terraform/applications/frontend/main.tf
+++ /dev/null
@@ -1,50 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-resource "google_app_engine_flexible_app_version" "frontend_app" {
- version_id = "v1"
- project = var.project_id
- service = "${var.service_name}-${var.environment}"
- runtime = "custom"
- delete_service_on_destroy = true
-
- liveness_check {
- path = ""
- }
-
- readiness_check {
- path = ""
- }
-
- manual_scaling {
- instances = 1
- }
-
- network {
- name = var.network_name
- subnetwork = var.subnetwork_name
- }
-
- deployment {
- container {
- image = "${var.docker_registry_address}/${var.docker_image_name}:${var.docker_image_tag}"
- }
- }
-}
-
diff --git a/playground/terraform/applications/frontend/output.tf b/playground/terraform/applications/frontend/output.tf
deleted file mode 100644
index e0dbab1f07a..00000000000
--- a/playground/terraform/applications/frontend/output.tf
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-output "front-url" {
- value = google_app_engine_flexible_app_version.frontend_app.service
-}
-
-
diff --git a/playground/terraform/applications/frontend/variables.tf b/playground/terraform/applications/frontend/variables.tf
deleted file mode 100644
index 24df9e6356d..00000000000
--- a/playground/terraform/applications/frontend/variables.tf
+++ /dev/null
@@ -1,55 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-variable "project_id" {
- description = "Project ID"
-}
-
-variable "docker_registry_address" {
- description = "Docker registry address"
-}
-
-variable "docker_image_name" {
- description = "Docker Image Name To Be Deployed"
- default = "beam_playground-frontend"
-}
-
-variable "docker_image_tag" {
- description = "Docker Image Tag To Be Deployed"
- default = "latest"
-}
-
-variable "service_name" {
- default = "frontend"
-}
-
-variable "network_name" {
- description = "VPC Name"
- default = "playground-vpc"
-}
-
-variable "subnetwork_name" {
- description = "Vpc Name"
- default = "playground-vpc"
-}
-
-variable "environment" {
- description = "prod,dev"
- default = "dev"
-}
diff --git a/playground/terraform/applications/main.tf b/playground/terraform/applications/main.tf
deleted file mode 100644
index 9c133e423fd..00000000000
--- a/playground/terraform/applications/main.tf
+++ /dev/null
@@ -1,80 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-
-data "terraform_remote_state" "playground-state" {
- backend = "gcs"
- config = {
- bucket = var.state_bucket
- prefix = var.state_prefix
- }
-}
-
-module "backend" {
-
- source = "./backend"
- project_id = var.project_id
- cache_address = data.terraform_remote_state.playground-state.outputs.playground_redis_ip
- docker_registry_address = data.terraform_remote_state.playground-state.outputs.docker-repository-root
- network_name = data.terraform_remote_state.playground-state.outputs.playground_network_name
- subnetwork_name = data.terraform_remote_state.playground-state.outputs.playground_subnetwork_name
- environment = var.environment
- docker_image_tag = var.docker_image_tag == "" ? var.environment : var.docker_image_tag
- docker_image_name = "${var.docker_image_name}-backend"
- cache_type = var.cache_type
-
- go_volume_size = var.go_volume_size
- go_cpu = var.go_cpu
- go_memory = var.go_memory
- go_max_instance = var.go_max_instance
- go_min_instance = var.go_min_instance
- java_volume_size = var.java_volume_size
- java_cpu = var.java_cpu
- java_memory = var.java_memory
- java_max_instance = var.java_max_instance
- java_min_instance = var.java_min_instance
- python_volume_size = var.python_volume_size
- python_cpu = var.python_cpu
- python_memory = var.python_memory
- python_max_instance = var.python_max_instance
- python_min_instance = var.python_min_instance
- router_volume_size = var.router_volume_size
- router_cpu = var.router_cpu
- router_memory = var.router_memory
- router_max_instance = var.router_max_instance
- router_min_instance = var.router_min_instance
- scio_volume_size = var.scio_volume_size
- scio_cpu = var.scio_cpu
- scio_memory = var.scio_memory
- scio_max_instance = var.scio_max_instance
- scio_min_instance = var.scio_min_instance
-}
-
-module "frontend" {
-
- source = "./frontend"
- project_id = var.project_id
- docker_registry_address = data.terraform_remote_state.playground-state.outputs.docker-repository-root
- network_name = data.terraform_remote_state.playground-state.outputs.playground_network_name
- subnetwork_name = data.terraform_remote_state.playground-state.outputs.playground_subnetwork_name
- environment = var.environment
- docker_image_tag = var.docker_image_tag == "" ? var.environment : var.docker_image_tag
- docker_image_name = "${var.docker_image_name}-frontend"
- service_name = var.frontend_service_name
-}
diff --git a/playground/terraform/applications/output.tf b/playground/terraform/applications/output.tf
deleted file mode 100644
index 9ed89830bf2..00000000000
--- a/playground/terraform/applications/output.tf
+++ /dev/null
@@ -1,42 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-output "go-server-url" {
- value = "https://${module.backend.go-server-url}-dot-${var.project_id}.${var.app_domain}/"
-}
-
-output "java-server-url" {
- value = "https://${module.backend.java-server-url}-dot-${var.project_id}.${var.app_domain}/"
-}
-
-output "python-server-url" {
- value = "https://${module.backend.python-server-url}-dot-${var.project_id}.${var.app_domain}/"
-}
-
-output "router-server-url" {
- value = "https://${module.backend.router-server-url}-dot-${var.project_id}.${var.app_domain}/"
-}
-
-output "scio-server-url" {
- value = "https://${module.backend.scio-server-url}-dot-${var.project_id}.${var.app_domain}/"
-}
-
-output "front-server-url" {
- value = "https://${module.frontend.front-url}-dot-${var.project_id}.${var.app_domain}/"
-}
diff --git a/playground/terraform/applications/variables.tf b/playground/terraform/applications/variables.tf
deleted file mode 100644
index 96b322c649b..00000000000
--- a/playground/terraform/applications/variables.tf
+++ /dev/null
@@ -1,244 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-variable "project_id" {
- description = "The GCP Project ID where Playground Applications will be created"
-}
-
-variable "environment" {
- description = "Environment name, e.g. prod,dev,beta"
- default = "dev"
-}
-
-variable "network_name" {
- description = "Vpc Name"
- default = "playground-vpc"
-}
-
-variable "subnetwork_name" {
- description = "Vpc Name"
- default = "playground-vpc"
-}
-
-variable "docker_image_tag" {
- description = "Docker Image Tag To Be Deployed"
- default = ""
-}
-
-variable "backend_service_name" {
- default = "backend"
-}
-
-variable "docker_image_name" {
- default = "beam_playground"
- description = "Base prefix for docker images"
-}
-
-variable "cache_type" {
- description = "remote or local. Set remote to use Redis and local to use in-memory application level cache"
- default = "remote"
-}
-
-variable "location" {
- description = "Location of App"
- default = "us-central"
-}
-
-variable "app_domain" {
- default = "uc.r.appspot.com"
-}
-
-# Frontend variables
-
-variable "frontend_service_name" {
- default = "frontend"
-}
-
-# SCIO variables
-
-variable "scio_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "scio_max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "scio_min_instance" {
- description = "Min count instance app"
- type = number
- default = 1
-}
-
-variable "scio_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "scio_cpu" {
- description = "CPU on instance"
- type = number
- default = 8
-}
-
-# Go variables
-
-variable "go_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "go_max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "go_min_instance" {
- description = "Min count instance app"
- type = number
- default = 1
-}
-
-variable "go_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "go_cpu" {
- description = "CPU on instance"
- type = number
- default = 8
-}
-
-# Python variables
-
-variable "python_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "python_max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "python_min_instance" {
- description = "Min count instance app"
- type = number
- default = 1
-}
-
-variable "python_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "python_cpu" {
- description = "CPU on instance"
- type = number
- default = 8
-}
-
-# Java variables
-
-variable "java_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "java_max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "java_min_instance" {
- description = "Min count instance app"
- type = number
- default = 1
-}
-
-variable "java_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "java_cpu" {
- description = "CPU on instance"
- type = number
- default = 8
-}
-
-# Router variables
-
-variable "router_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-
-variable "router_max_instance" {
- description = "Max count instance app"
- type = number
- default = 3
-}
-
-variable "router_min_instance" {
- description = "Min count instance app"
- type = number
- default = 1
-}
-
-variable "router_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 4
-}
-
-variable "router_cpu" {
- description = "CPU on instance"
- type = number
- default = 2
-}
-
-
-variable "create_default_service" {
- description = "Whether or not to create a default app engine service"
- type = bool
- default = true
-}
-
-variable "state_bucket" {}
-variable "state_prefix" {
- default = ""
-}
diff --git a/playground/terraform/build.gradle.kts b/playground/terraform/build.gradle.kts
index a7eed0c13b5..8e1e5b48fca 100644
--- a/playground/terraform/build.gradle.kts
+++ b/playground/terraform/build.gradle.kts
@@ -43,7 +43,7 @@ plugins {
}
terraformPlugin {
- terraformVersion.set("1.0.9")
+ terraformVersion.set("1.4.2")
}
tasks {
@@ -91,88 +91,6 @@ tasks {
)
}
- /* deploy all App */
- register<TerraformTask>("terraformApplyApp") {
- var project_id = "unknown"
- var environment = "unknown"
- if (project.hasProperty("project_id")) {
- project_id = project.property("project_id") as String
- }
- if (project.hasProperty("project_environment")) {
- environment = project.property("project_environment") as String
- }
- var docker_tag = if (project.hasProperty("docker-tag")) {
- project.property("docker-tag") as String
- } else {
- environment
- }
- args(
- "apply",
- "-auto-approve",
- "-lock=false",
- "-target=module.applications",
- "-var=project_id=$project_id",
- "-var=environment=$environment",
- "-var=docker_image_tag=$docker_tag",
- if (file("./environment/$environment/terraform.tfvars").exists()) {
- "-var-file=./environment/$environment/terraform.tfvars"
- } else {
- "-no-color"
- }
- )
- }
-
- /* deploy App - Only all services for backend */
- register<TerraformTask>("terraformApplyAppBack") {
- var environment = "unknown"
- if (project.hasProperty("project_environment")) {
- environment = project.property("project_environment") as String
- }
- args(
- "apply",
- "-auto-approve",
- "-lock=false",
- "-target=module.applications.module.backend",
- "-var=environment=$environment",
- if (file("./environment/$environment/terraform.tfvars").exists()) {
- "-var-file=./environment/$environment/terraform.tfvars"
- } else {
- "-no-color"
- }
- )
- }
-
- /* deploy App - Only services for frontend */
- register<TerraformTask>("terraformApplyAppFront") {
- var project_id = "unknown"
- var environment = "unknown"
- if (project.hasProperty("project_id")) {
- project_id = project.property("project_id") as String
- }
- if (project.hasProperty("project_environment")) {
- environment = project.property("project_environment") as String
- }
- var docker_tag = if (project.hasProperty("docker-tag")) {
- project.property("docker-tag") as String
- } else {
- environment
- }
- args(
- "apply",
- "-auto-approve",
- "-lock=false",
- "-target=module.applications.module.frontend",
- "-var=project_id=$project_id",
- "-var=environment=$environment",
- "-var=docker_image_tag=$docker_tag",
- if (file("./environment/$environment/terraform.tfvars").exists()) {
- "-var-file=./environment/$environment/terraform.tfvars"
- } else {
- "-no-color"
- }
- )
- }
-
/* build only Infrastructurte */
register<TerraformTask>("terraformApplyInf") {
var environment = "unknown"
@@ -292,21 +210,6 @@ tasks.register("indexcreate") {
}
}
-/* build, push, deploy Frontend app */
-tasks.register("deployFrontend") {
- group = "deploy"
- description = "deploy Frontend app"
- val read = tasks.getByName("readState")
- val push = tasks.getByName("pushFront")
- val deploy = tasks.getByName("terraformApplyAppFront")
- dependsOn(read)
- Thread.sleep(10)
- push.mustRunAfter(read)
- deploy.mustRunAfter(push)
- dependsOn(push)
- dependsOn(deploy)
-}
-
tasks.register<TerraformTask>("setPlaygroundStaticIpAddress") {
group = "deploy"
@@ -410,6 +313,7 @@ tasks.register("takeConfig") {
val proj = project.rootProject.extra["playground_gke_project"]
val registry = project.rootProject.extra["docker-repository-root"]
val ipaddrname = project.rootProject.extra["playground_static_ip_address_name"]
+ val datastore_name = if (project.hasProperty("datastore-namespace")) (project.property("datastore-namespace") as String) else ""
file.appendText(
"""
@@ -419,6 +323,7 @@ project_id: ${proj}
registry: ${registry}
static_ip_name: ${ipaddrname}
tag: $d_tag
+datastore_name: ${datastore_name}
dns_name: ${dns_name}
"""
)
@@ -428,14 +333,15 @@ dns_name: ${dns_name}
tasks.register("helmRelease") {
group = "deploy"
val modulePath = project(":playground").projectDir.absolutePath
- val hdir = File("$modulePath/infrastructure/helm-playground/")
- doLast {
- exec {
- executable("helm")
- args("install", "playground", "$hdir")
- }
+ val helmdir = File("$modulePath/infrastructure/helm-playground/")
+ doLast{
+ exec {
+ executable("helm")
+ args("upgrade", "--install", "playground", "$helmdir")
+ }
}
}
+
tasks.register("gkebackend") {
group = "deploy"
val initTask = tasks.getByName("terraformInit")
@@ -456,4 +362,3 @@ tasks.register("gkebackend") {
indexcreateTask.mustRunAfter(pushFrontTask)
helmTask.mustRunAfter(indexcreateTask)
}
-
diff --git a/playground/terraform/infrastructure/api_enable/variables.tf b/playground/terraform/infrastructure/api_enable/variables.tf
index f2cdcbf2874..f0c0e5823f3 100644
--- a/playground/terraform/infrastructure/api_enable/variables.tf
+++ b/playground/terraform/infrastructure/api_enable/variables.tf
@@ -23,5 +23,5 @@ variable "project_id" {
variable "services" {
description = "Enable necessary APIs in GCP"
- default = ["cloudresourcemanager.googleapis.com","iam.googleapis.com","compute.googleapis.com","appengine.googleapis.com","artifactregistry.googleapis.com","redis.googleapis.com"]
+ default = ["cloudresourcemanager.googleapis.com","iam.googleapis.com","compute.googleapis.com","appengine.googleapis.com","artifactregistry.googleapis.com","redis.googleapis.com","dns.googleapis.com","certificatemanager.googleapis.com"]
}
diff --git a/playground/terraform/infrastructure/appengine/main.tf b/playground/terraform/infrastructure/appengine/main.tf
index dff92a08274..f8276a6f864 100644
--- a/playground/terraform/infrastructure/appengine/main.tf
+++ b/playground/terraform/infrastructure/appengine/main.tf
@@ -13,13 +13,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+
resource "google_app_engine_application" "app_playground" {
+ count = var.skip_appengine_deploy ? 0 : 1
project = var.project_id
- location_id = var.region == "us-central1" ? var.location_id_us : var.region
+ location_id = var.region == "us-central1" ? var.location_id_us : (var.region == "europe-west1") ? var.location_id_eu : var.region
database_type = "CLOUD_DATASTORE_COMPATIBILITY"
}
resource "google_project_service" "firestore" {
+ count = var.skip_appengine_deploy ? 0 : 1
project = var.project_id
service = "firestore.googleapis.com"
disable_dependent_services = true
diff --git a/playground/terraform/infrastructure/appengine/variables.tf b/playground/terraform/infrastructure/appengine/variables.tf
index 6f1f6c7fefe..54b887d43bb 100644
--- a/playground/terraform/infrastructure/appengine/variables.tf
+++ b/playground/terraform/infrastructure/appengine/variables.tf
@@ -36,3 +36,13 @@ variable "location_id_us" {
description = "Location of App"
default = "us-central"
}
+
+variable "location_id_eu" {
+ description = "Location of App"
+ default = "europe-west"
+}
+
+variable "skip_appengine_deploy" {
+ type = bool
+ default = false
+}
diff --git a/playground/terraform/infrastructure/firewall/firewall.tf b/playground/terraform/infrastructure/firewall/firewall.tf
new file mode 100644
index 00000000000..cea4fad4c08
--- /dev/null
+++ b/playground/terraform/infrastructure/firewall/firewall.tf
@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+resource "google_compute_firewall" "playground-firewall-deny-egress" {
+ name = "${var.network_name}-deny-egress"
+ network = var.network_name
+ direction = "EGRESS"
+ priority = 1001
+ deny {
+ protocol = "all"
+ }
+ destination_ranges = ["0.0.0.0/0"]
+ target_tags = ["beam-playground"]
+}
+
+resource "google_compute_firewall" "playground-firewall-allow-controlplane" {
+ name = "${var.network_name}-allow-controlplane"
+ network = var.network_name
+ direction = "EGRESS"
+ priority = 1000
+ allow {
+ protocol = "all"
+ }
+ destination_ranges = [var.gke_controlplane_cidr]
+ target_tags = ["beam-playground"]
+}
+
+resource "google_compute_firewall" "playground-firewall-allow-dns" {
+ name = "${var.network_name}-allow-dns"
+ network = var.network_name
+ direction = "EGRESS"
+ priority = 1000
+ allow {
+ protocol = "tcp"
+ ports = ["53"]
+
+ }
+ allow {
+ protocol = "udp"
+ ports = ["53"]
+ }
+ destination_ranges = ["0.0.0.0/0"]
+ target_tags = ["beam-playground"]
+}
+
+# Allows for private google access as per:
+# https://cloud.google.com/vpc/docs/configure-private-google-access#config
+resource "google_compute_firewall" "playground-firewall-allow-privateapi" {
+ name = "${var.network_name}-allow-privateapi"
+ network = var.network_name
+ direction = "EGRESS"
+ priority = 1000
+ allow {
+ protocol = "all"
+ }
+
+ destination_ranges = [local.private_api_range]
+ target_tags = ["beam-playground"]
+}
+
+
+resource "google_compute_firewall" "playground-firewall-allow-redis" {
+ name = "${var.network_name}-allow-redis"
+ network = var.network_name
+ direction = "EGRESS"
+ priority = 1000
+ allow {
+ protocol = "all"
+ }
+
+ destination_ranges = [var.redis_ip]
+ target_tags = ["beam-playground"]
+}
\ No newline at end of file
diff --git a/playground/infrastructure/helm-playground/templates/backendconf.yml b/playground/terraform/infrastructure/firewall/locals.tf
similarity index 82%
copy from playground/infrastructure/helm-playground/templates/backendconf.yml
copy to playground/terraform/infrastructure/firewall/locals.tf
index 725a9e13b13..12b0b1d1cd9 100644
--- a/playground/infrastructure/helm-playground/templates/backendconf.yml
+++ b/playground/terraform/infrastructure/firewall/locals.tf
@@ -1,4 +1,3 @@
-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -14,11 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-apiVersion: cloud.google.com/v1
-kind: BackendConfig
-metadata:
- name: bkend-health
-spec:
- healthCheck:
- type: HTTP
- requestPath: /liveness
+# Allows for private google access as per:
+# https://cloud.google.com/vpc/docs/configure-private-google-access#config
+locals {
+ private_api_range = "199.36.153.8/30"
+}
diff --git a/playground/terraform/applications/backend/backend-java/output.tf b/playground/terraform/infrastructure/firewall/variables.tf
similarity index 75%
rename from playground/terraform/applications/backend/backend-java/output.tf
rename to playground/terraform/infrastructure/firewall/variables.tf
index 1974c191018..146acc82062 100644
--- a/playground/terraform/applications/backend/backend-java/output.tf
+++ b/playground/terraform/infrastructure/firewall/variables.tf
@@ -16,8 +16,14 @@
# specific language governing permissions and limitations
# under the License.
#
-
-output "server-url" {
- value = google_app_engine_flexible_app_version.backend_app_java.service
+variable "network_name" {
+ description = "Name of VPC to be created"
+ default = "playground-network"
}
+variable "gke_controlplane_cidr" {
+ description = "CIDR for gke controlplane rule"
+}
+variable "redis_ip" {
+ description = "IP address for Redis instance"
+}
\ No newline at end of file
diff --git a/playground/terraform/infrastructure/gke/main.tf b/playground/terraform/infrastructure/gke/main.tf
index 3e9dd990e67..b82ef9f9a29 100644
--- a/playground/terraform/infrastructure/gke/main.tf
+++ b/playground/terraform/infrastructure/gke/main.tf
@@ -25,8 +25,13 @@ resource "google_container_cluster" "playground-gke" {
network = var.network
subnetwork = var.subnetwork
remove_default_node_pool = true
-}
+ private_cluster_config {
+ enable_private_nodes = true
+ enable_private_endpoint = false
+ master_ipv4_cidr_block = var.control_plane_cidr
+}
+}
resource "google_container_node_pool" "playground-node-pool" {
name = "playground-node-pool"
cluster = google_container_cluster.playground-gke.name
diff --git a/playground/terraform/infrastructure/gke/output.tf b/playground/terraform/infrastructure/gke/output.tf
index 4a1fd941720..fa5c4ffb4af 100644
--- a/playground/terraform/infrastructure/gke/output.tf
+++ b/playground/terraform/infrastructure/gke/output.tf
@@ -24,3 +24,7 @@ output "gke_name" {
output "playground_gke_project" {
value = google_container_cluster.playground-gke.project
}
+
+output "control_plane_cidr" {
+ value = var.control_plane_cidr
+}
diff --git a/playground/terraform/infrastructure/gke/variables.tf b/playground/terraform/infrastructure/gke/variables.tf
index 3b3b7369d51..d3b27feb6b1 100644
--- a/playground/terraform/infrastructure/gke/variables.tf
+++ b/playground/terraform/infrastructure/gke/variables.tf
@@ -23,7 +23,7 @@ variable "project_id" {
variable "machine_type" {
description = "Node pool machine types , for prod set c2d-highcpu-16"
- default = "e2-standard-4"
+ default = "e2-standard-8"
}
variable "service_account_email" {
description = "Service account email"
@@ -54,4 +54,9 @@ variable "min_count" {
variable "max_count" {
description = "Max cluster node count"
default = 6
-}
\ No newline at end of file
+}
+variable "control_plane_cidr" {
+ description = "CIDR block for GKE controlplane"
+ default = "10.129.0.0/28"
+}
+
diff --git a/playground/terraform/infrastructure/ip_address/main.tf b/playground/terraform/infrastructure/ip_address/main.tf
index 4f4ee35e9c9..3e2a8f3524a 100644
--- a/playground/terraform/infrastructure/ip_address/main.tf
+++ b/playground/terraform/infrastructure/ip_address/main.tf
@@ -18,5 +18,5 @@
#
resource "google_compute_global_address" "pg-ip" {
- name = "pg-static-ip"
-}
\ No newline at end of file
+ name = var.ip-address-name
+}
diff --git a/playground/terraform/infrastructure/ip_address/variables.tf b/playground/terraform/infrastructure/ip_address/variables.tf
index 661b0cba774..42f60ed7ae3 100644
--- a/playground/terraform/infrastructure/ip_address/variables.tf
+++ b/playground/terraform/infrastructure/ip_address/variables.tf
@@ -17,4 +17,7 @@
# under the License.
#
-
+variable "ip-address-name" {
+ description = "Static IP address name"
+ default = "pg-static-ip"
+}
diff --git a/playground/terraform/infrastructure/main.tf b/playground/terraform/infrastructure/main.tf
index fa9353b72da..edb32a36857 100644
--- a/playground/terraform/infrastructure/main.tf
+++ b/playground/terraform/infrastructure/main.tf
@@ -33,6 +33,14 @@ module "network" {
subnetwork_name = var.subnetwork_name
}
+module "firewall" {
+ depends_on = [module.network, module.memorystore, module.gke]
+ source = "./firewall"
+ network_name = module.network.playground_network_name
+ gke_controlplane_cidr = module.gke.control_plane_cidr
+ redis_ip = module.memorystore.redis_ip
+}
+
module "artifact_registry" {
depends_on = [module.setup, module.api_enable, module.ip_address]
source = "./artifact_registry"
@@ -73,16 +81,31 @@ module "gke" {
module "ip_address" {
source = "./ip_address"
depends_on = [module.setup, module.api_enable]
+ ip-address-name = var.ip-address-name
}
module "appengine" {
- depends_on = [module.setup, module.api_enable, module.ip_address]
- source = "./appengine"
- project_id = var.project_id
- region = var.region
+ depends_on = [module.setup, module.api_enable, module.ip_address]
+ source = "./appengine"
+ project_id = var.project_id
+ region = var.region
+ skip_appengine_deploy = var.skip_appengine_deploy
}
module "api_enable" {
source = "./api_enable"
project_id = var.project_id
}
+
+
+module "private_dns" {
+ source = "./private_dns"
+ project_id = var.project_id
+ network_id = module.network.playground_network_id
+ network_name = module.network.playground_network_name
+ private_zones = [
+ "gcr.io",
+ "pkg.dev",
+ "cloud.google.com"
+ ]
+}
\ No newline at end of file
diff --git a/playground/terraform/infrastructure/memorystore/variables.tf b/playground/terraform/infrastructure/memorystore/variables.tf
index 49e80637b07..127f72e9ac1 100644
--- a/playground/terraform/infrastructure/memorystore/variables.tf
+++ b/playground/terraform/infrastructure/memorystore/variables.tf
@@ -48,7 +48,7 @@ variable "tier" {
variable "replicas_mode" {
description = "If enabled read endpoint will be provided and the instance can scale up and down the number of replicas"
- default = "READ_REPLICAS_ENABLED"
+ default = "READ_REPLICAS_DISABLED"
}
variable "network" {
@@ -63,7 +63,7 @@ variable "subnetwork" {
variable "replica_count" {
type = number
description = "Redis's replica count"
- default = 1
+ default = 0
}
variable "memory_size_gb" {
diff --git a/playground/terraform/infrastructure/private_dns/googleapis_com.tf b/playground/terraform/infrastructure/private_dns/googleapis_com.tf
new file mode 100644
index 00000000000..93a95ec50a2
--- /dev/null
+++ b/playground/terraform/infrastructure/private_dns/googleapis_com.tf
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+resource "google_dns_managed_zone" "private-zone-private-googleapis" {
+ project = var.project_id
+
+ name = "${var.network_name}googleapis-com"
+ dns_name = "googleapis.com."
+ description = "Private GoogleApi Zone"
+
+ visibility = "private"
+
+ private_visibility_config {
+ networks {
+ network_url = var.network_id
+ }
+ }
+}
+
+resource "google_dns_record_set" "a-private-googleapis" {
+ name = "private.googleapis.com."
+ type = "A"
+ ttl = 300
+
+ managed_zone = google_dns_managed_zone.private-zone-private-googleapis.name
+
+ rrdatas = local.private_api_ips
+}
+
+resource "google_dns_record_set" "cname-private-googleapis" {
+ name = "*.googleapis.com."
+ managed_zone = google_dns_managed_zone.private-zone-private-googleapis.name
+ type = "CNAME"
+ ttl = 300
+ rrdatas = ["private.googleapis.com."]
+}
diff --git a/playground/terraform/applications/backend/backend-python/output.tf b/playground/terraform/infrastructure/private_dns/locals.tf
similarity index 78%
rename from playground/terraform/applications/backend/backend-python/output.tf
rename to playground/terraform/infrastructure/private_dns/locals.tf
index 877b1491735..82d6b09ef8c 100644
--- a/playground/terraform/applications/backend/backend-python/output.tf
+++ b/playground/terraform/infrastructure/private_dns/locals.tf
@@ -17,7 +17,8 @@
# under the License.
#
-output "server-url" {
- value = google_app_engine_flexible_app_version.backend_app_python.service
-}
-
+# Allows for private google access as per:
+# https://cloud.google.com/vpc/docs/configure-private-google-access#config
+locals {
+ private_api_ips = ["199.36.153.8", "199.36.153.9", "199.36.153.10", "199.36.153.11"]
+}
\ No newline at end of file
diff --git a/playground/terraform/applications/default/variables.tf b/playground/terraform/infrastructure/private_dns/variables.tf
similarity index 70%
rename from playground/terraform/applications/default/variables.tf
rename to playground/terraform/infrastructure/private_dns/variables.tf
index 565dc48e77e..0b78aaf5459 100644
--- a/playground/terraform/applications/default/variables.tf
+++ b/playground/terraform/infrastructure/private_dns/variables.tf
@@ -17,17 +17,19 @@
# under the License.
#
-variable "project_id" {
- description = "The GCP Project ID where Playground Applications will be created"
+variable "network_id" {
+ description = "Network ID to use private dns zone in"
+}
+
+variable "network_name" {
+ description = "Network name to create name for zone"
}
-variable "location" {
- description = "Location of App"
- default = "us-central"
+variable "private_zones" {
+ description = "List of private zones to point to private.googleapis.com"
+ type = set(string)
}
-variable "create_default_service" {
- description = "Whether or not to create a default app engine service"
- type = bool
- default = true
-}
\ No newline at end of file
+variable "project_id" {
+ description = "Project ID to use private dns zone in"
+}
diff --git a/playground/terraform/infrastructure/private_dns/zones.tf b/playground/terraform/infrastructure/private_dns/zones.tf
new file mode 100644
index 00000000000..6a28ed7cb9a
--- /dev/null
+++ b/playground/terraform/infrastructure/private_dns/zones.tf
@@ -0,0 +1,63 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+resource "google_dns_managed_zone" "private-zone-apis" {
+
+ for_each = var.private_zones
+
+ project = var.project_id
+
+ name = "${var.network_name}-${replace(each.key,".","-")}"
+ dns_name = format("%s%s",each.key,".")
+ description = "Private ${each.key} Zone"
+
+ visibility = "private"
+
+ private_visibility_config {
+ networks {
+ network_url = var.network_id
+ }
+ }
+}
+
+resource "google_dns_record_set" "a-private-zone" {
+
+ for_each = google_dns_managed_zone.private-zone-apis
+
+ name = each.value.dns_name
+ type = "A"
+ ttl = 300
+
+ managed_zone = each.value.name
+
+ rrdatas = local.private_api_ips
+}
+
+resource "google_dns_record_set" "cname-private-zone" {
+
+ for_each = google_dns_managed_zone.private-zone-apis
+
+ name = "*.${each.value.dns_name}"
+ managed_zone = each.value.name
+ type = "CNAME"
+ ttl = 300
+ rrdatas = [each.value.dns_name]
+}
+
+
diff --git a/playground/terraform/infrastructure/variables.tf b/playground/terraform/infrastructure/variables.tf
index 168c70c85e5..80fddf807c9 100644
--- a/playground/terraform/infrastructure/variables.tf
+++ b/playground/terraform/infrastructure/variables.tf
@@ -94,12 +94,12 @@ variable "redis_name" {
variable "redis_tier" {
description = "Tier of Redis"
- default = "STANDARD_HA"
+ default = "BASIC"
}
variable "redis_replica_count" {
description = "Redis's replica count"
- default = 1
+ default = 0
}
variable "redis_memory_size_gb" {
@@ -109,7 +109,7 @@ variable "redis_memory_size_gb" {
variable "read_replicas_mode" {
description = "Read replica mode. Can only be specified when trying to create the instance."
- default = "READ_REPLICAS_ENABLED"
+ default = "READ_REPLICAS_DISABLED"
}
#NETWORK
@@ -129,11 +129,23 @@ variable "network_region" {
default = "us-central1"
}
+variable "ip-address-name" {
+ description = "Static IP address name"
+ default = "pg-static-ip"
+}
+
+# APPENGINE
+variable "skip_appengine_deploy" {
+ description = "AppEngine enabled"
+ type = bool
+ default = false
+}
+
# GKE
variable "gke_machine_type" {
description = "Node pool machine types"
- default = "e2-standard-4"
+ default = "e2-standard-8"
}
variable "gke_name" {
@@ -157,4 +169,4 @@ variable "min_count" {
variable "max_count" {
description = "Max cluster node count"
default = 6
-}
\ No newline at end of file
+}
diff --git a/playground/terraform/main.tf b/playground/terraform/main.tf
index ffbedca10ba..e852ba9ffc6 100644
--- a/playground/terraform/main.tf
+++ b/playground/terraform/main.tf
@@ -24,7 +24,8 @@ module "infrastructure" {
region = var.region
network_region = var.region
redis_region = var.region
- location = var.location
+ location = var.zone
+ service_account_id = var.service_account_id
#Artifact Registry
repository_id = var.repository_id
repository_location = var.region
@@ -36,51 +37,14 @@ module "infrastructure" {
redis_memory_size_gb = var.redis_memory_size_gb
#NETWORK
network_name = var.network_name
+ ip-address-name = var.ip-address-name
+ subnetwork_name = var.subnetwork_name
#GKE
gke_machine_type = var.gke_machine_type
gke_name = var.gke_name
- gke_location = var.location
+ gke_location = var.zone
service_account = var.service_account
-}
-
-module "applications" {
- source = "./applications"
- project_id = var.project_id
- environment = var.environment
- docker_image_name = var.docker_image_name
- docker_image_tag = var.docker_image_tag
- backend_service_name = var.backend_service_name
- frontend_service_name = var.frontend_service_name
- cache_type = var.cache_type
-
- go_volume_size = var.go_volume_size
- go_cpu = var.go_cpu
- go_memory = var.go_memory
- go_max_instance = var.go_max_instance
- go_min_instance = var.go_min_instance
- java_volume_size = var.java_volume_size
- java_cpu = var.java_cpu
- java_memory = var.java_memory
- java_max_instance = var.java_max_instance
- java_min_instance = var.java_min_instance
- python_volume_size = var.python_volume_size
- python_cpu = var.python_cpu
- python_memory = var.python_memory
- python_max_instance = var.python_max_instance
- python_min_instance = var.python_min_instance
- router_volume_size = var.router_volume_size
- router_cpu = var.router_cpu
- router_memory = var.router_memory
- router_max_instance = var.router_max_instance
- router_min_instance = var.router_min_instance
- scio_volume_size = var.scio_volume_size
- scio_cpu = var.scio_cpu
- scio_memory = var.scio_memory
- scio_max_instance = var.scio_max_instance
- scio_min_instance = var.scio_min_instance
-
- # location = var.application_location
- create_default_service = var.create_default_service
- state_bucket = var.state_bucket
- state_prefix = var.state_prefix
+ skip_appengine_deploy = var.skip_appengine_deploy
+ min_count = var.min_count
+ max_count = var.max_count
}
diff --git a/playground/terraform/variables.tf b/playground/terraform/variables.tf
index 9d5c3ba9e4a..a8b3c9144ab 100644
--- a/playground/terraform/variables.tf
+++ b/playground/terraform/variables.tf
@@ -31,13 +31,17 @@ variable "region" {
description = "Infrastructure Region"
}
+variable "zone" {
+ description = "Infrastructure Zone"
+}
+
# Infrastructure variables
#GKE
variable "gke_machine_type" {
description = "Node pool machine types"
- default = "e2-standard-4"
+ default = "e2-standard-8"
}
variable "gke_name" {
@@ -66,6 +70,30 @@ variable "repository_id" {
default = "playground-repository"
}
+#IAM
+variable "service_account_id" {
+ description = "Service account ID"
+ default = "beam-playground"
+}
+
+#Network
+variable "ip-address-name" {
+ description = "Static IP address name"
+ default = "pg-static-ip"
+}
+
+variable "subnetwork_name" {
+ description = "Name of VPC to be created"
+ default = "playground-vpc-sub"
+}
+
+#AppEngine Flag
+variable "skip_appengine_deploy" {
+ description = "AppEngine enabled"
+ type = bool
+ default = false
+}
+
#Redis
variable "redis_version" {
@@ -80,12 +108,12 @@ variable "redis_name" {
variable "redis_tier" {
description = "Tier of Redis"
- default = "STANDARD_HA"
+ default = "BASIC"
}
variable "redis_replica_count" {
description = "Redis's replica count"
- default = 1
+ default = 0
}
variable "redis_memory_size_gb" {
@@ -116,185 +144,3 @@ variable "docker_image_name" {
default = "beam_playground"
description = "Base prefix for docker images"
}
-
-# Frontend variables
-
-variable "frontend_service_name" {
- default = "frontend"
-}
-
-# Backend variables
-
-variable "cache_type" {
- description = "remote or local. Set remote to use Redis and local to use in-memory application level cache"
- default = "remote"
-}
-variable "backend_service_name" {
- default = "backend"
-}
-
-# For scio backend service
-variable "scio_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "scio_max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "scio_min_instance" {
- description = "Min count instance app"
- type = number
- default = 1
-}
-
-variable "scio_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "scio_cpu" {
- description = "CPU on instance"
- type = number
- default = 8
-}
-
-# For go backend service
-variable "go_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "go_max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "go_min_instance" {
- description = "Min count instance app"
- type = number
- default = 1
-}
-
-variable "go_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "go_cpu" {
- description = "CPU on instance"
- type = number
- default = 8
-}
-
-# For python backend service
-variable "python_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "python_max_instance" {
- description = "Max count instance app"
- type = number
- default = 7
-}
-
-variable "python_min_instance" {
- description = "Min count instance app"
- type = number
- default = 1
-}
-
-variable "python_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "python_cpu" {
- description = "CPU on instance"
- type = number
- default = 8
-}
-
-# For java backend service
-variable "java_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "java_max_instance" {
- description = "Max count instance app"
- type = number
- default =7
-}
-
-variable "java_min_instance" {
- description = "Min count instance app"
- type = number
- default = 1
-}
-
-variable "java_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 16
-}
-
-variable "java_cpu" {
- description = "CPU on instance"
- type = number
- default = 8
-}
-
-# For route backend service
-variable "router_volume_size" {
- description = "Size of the in memory file system to be used by the application, in GB"
- type = number
- default = 1
-}
-
-variable "router_max_instance" {
- description = "Max count instance app"
- type = number
- default = 3
-}
-
-variable "router_min_instance" {
- description = "Min count instance app"
- type = number
- default = 1
-}
-
-variable "router_memory" {
- description = "Memory on instance in GB, 0.9-6.9 on ONE CPU"
- type = number
- default = 4
-}
-
-variable "router_cpu" {
- description = "CPU on instance"
- type = number
- default = 2
-}
-
-variable "state_bucket" {
- description = "GCP bucket that used to store terraform state"
- default = "beam_playground_terraform"
-}
-
-variable "state_prefix" {
- description = "terraform state prefix on GCP"
- default = ""
-}
-variable "location" {}
diff --git a/settings.gradle.kts b/settings.gradle.kts
index 75f56e6faae..f1733d538c4 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -145,7 +145,7 @@ include(":sdks:java:extensions:join-library")
include(":sdks:java:extensions:ml")
include(":sdks:java:extensions:protobuf")
include(":sdks:java:extensions:python")
-include("sdks:java:extensions:sbe")
+include(":sdks:java:extensions:sbe")
include(":sdks:java:extensions:schemaio-expansion-service")
include(":sdks:java:extensions:sketching")
include(":sdks:java:extensions:sorter")