[airflow-ci-infra] branch main updated: Disable shipping logs to CloudWatch

[as...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ash pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git


The following commit(s) were added to refs/heads/main by this push:
     new 8d7b211  Disable shipping logs to CloudWatch
8d7b211 is described below

commit 8d7b211dd62d9f5ace6e3cad230ca4479866a56f
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Fri Jan 21 20:15:12 2022 +0000

    Disable shipping logs to CloudWatch
    
    This was useful when we were debugging the "communication lost with
    instance" but we haven't seen those in months, and CloudWatch Logs
    accounts for 10% of our monthly AWS spend(!) and we just don't need it
    anymore
    
    I have included some previously un-pushed changes to the Vector config
    to drop a few of the more common and less interesting lines. Useful for
    posterity.
---
 cloud-init.yml                             |  5 +++--
 github-runner-ami/packer/files/vector.toml | 12 ++++++++++--
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/cloud-init.yml b/cloud-init.yml
index 183a6f4..bc69e87 100644
--- a/cloud-init.yml
+++ b/cloud-init.yml
@@ -1,5 +1,5 @@
 #cloud-config
-# 2021-05-17 09:21+01:00
+# 2022-01-21 19:31 BMT
 
 mounts:
   - [tmpfs, /tmp, tmpfs, "defaults,noatime,size=10%"]
@@ -15,5 +15,6 @@ runcmd:
       set -eu -o pipefail
       echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/environment
       echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/default/vector
-      systemctl enable --now vector
+      #aws logs create-log-stream --log-group-name GitHubRunners --log-stream-name $(hostname) || true
+      #systemctl enable --now vector
       systemctl enable --now actions.runner
diff --git a/github-runner-ami/packer/files/vector.toml b/github-runner-ami/packer/files/vector.toml
index 81112e6..f9f3308 100644
--- a/github-runner-ami/packer/files/vector.toml
+++ b/github-runner-ami/packer/files/vector.toml
@@ -31,8 +31,8 @@ data_dir = "/var/lib/vector"
   fields = ["_CAP_EFFECTIVE", "_SYSTEMD_SLICE", "_SYSTEMD_CGROUP",
     "_SYSTEMD_INVOCATION_ID", "_SELINUX_CONTEXT", "_COMM", "_BOOT_ID",
     "_MACHINE_ID", "_STREAM_ID", "_PID", "_GID", "_UID","_TRANSPORT",
-    "__MONOTONIC_TIMESTAMP", "SYSLOG_IDENTIFIER", "PRIORITY",
-    "source_type"]
+    "__MONOTONIC_TIMESTAMP", "SYSLOG_IDENTIFIER", "SYSLOG_TIMESTAMP",
+    "PRIORITY", "_EXE", "source_type"]
 
 [sources.runner-logs]
   type = "file"
@@ -64,6 +64,9 @@ data_dir = "/var/lib/vector"
       !match!(.message, r'Try to append \d+ batches web console lines for record')
     } else if .logger == "HostContext" {
       !starts_with!(.message, "Well known directory")
+    } else if .logger == "JobDispatcher" {
+      !starts_with!(.message, "Successfully renew job request")
+    }
     } else {
       true
     }
@@ -76,12 +79,17 @@ data_dir = "/var/lib/vector"
 [transforms.grok-job-logs]
   type = "remap"
   inputs = ["job-logs"]
+  drop_on_abort = true
   source = '''
     structured, err = parse_grok(.message, "%{TIMESTAMP_ISO8601:timestamp} %{GREEDYDATA:message}")
 
     if err == null {
       . = merge(., structured)
       .type = "job-output"
+
+      if length(.message) == 0 ?? true {
+        abort # Don't send empty messages to cloud watch!
+      }
     }
   '''