From 11fa03d8575136d872fd79bf1d7a92e19c766836 Mon Sep 17 00:00:00 2001 From: Rich Turner <7072278+richturner@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:51:17 +0000 Subject: [PATCH 1/6] Fix broken cfn-hup service in cloudformation-create-ec2.yml and add docker metric --- .ci_cd/aws/cloudformation-create-ec2.yml | 131 +++++++++++++++++++++-- 1 file changed, 121 insertions(+), 10 deletions(-) diff --git a/.ci_cd/aws/cloudformation-create-ec2.yml b/.ci_cd/aws/cloudformation-create-ec2.yml index ecb3955b47..6f898a54e4 100644 --- a/.ci_cd/aws/cloudformation-create-ec2.yml +++ b/.ci_cd/aws/cloudformation-create-ec2.yml @@ -235,6 +235,7 @@ Resources: - env_vars - dns_updater - s3_backup_service + - docker_metrics_service - deployment_local_dir - daily_cleanup @@ -581,7 +582,6 @@ Resources: journalctl --vacuum-time=7d "/etc/cron.d/cleanup": content: | - #!/bin/bash 0 3 * * * root /usr/local/bin/daily-cleanup commands: 01_chmod: @@ -592,7 +592,7 @@ Resources: commands: 01_create_dir: command: | - mkdir /deployment.local + mkdir -p /deployment.local chown ec2-user:ec2-user /deployment.local s3_backup_service: @@ -651,6 +651,81 @@ Resources: systemctl start s3-backup.service systemctl enable s3-backup.timer + docker_metrics_service: + files: + "/usr/local/bin/push-docker-metrics": + content: !Sub | + #!/bin/bash + set -euo pipefail + + # ------------------------------------------------------------------------------------------------------ + # + # Script for pushing max Docker container memory percentage to CloudWatch + # + # ------------------------------------------------------------------------------------------------------ + + NAMESPACE="Custom/Docker" + + MAX_MEM_PCT=$(docker stats --no-stream --format "{{.MemPerc}}" \ + | tr -d '%' \ + | grep -E '^[0-9]+(\.[0-9]+)?$' \ + | sort -nr \ + | head -n 1 || true) + + if [ -z "$MAX_MEM_PCT" ]; then + exit 0 + fi + + aws cloudwatch put-metric-data \ + --region "${AWS::Region}" \ + --namespace "$NAMESPACE" \ + --metric-name "MaxContainerMemoryPercent" \ + --dimensions "InstanceName=${Host}" \ + --value "$MAX_MEM_PCT" \ + --unit Percent + + "/lib/systemd/system/docker-metrics.service": + content: | + [Unit] + Description=Push max Docker container memory metric to CloudWatch + Wants=docker-metrics.timer + After=docker.service + + [Service] + Type=oneshot + ExecStart=/usr/local/bin/push-docker-metrics + + [Install] + WantedBy=multi-user.target + + "/lib/systemd/system/docker-metrics.timer": + content: | + [Unit] + Description=Run Docker CloudWatch metrics push every minute + Requires=docker-metrics.service + + [Timer] + Unit=docker-metrics.service + OnBootSec=1min + OnUnitActiveSec=1min + AccuracySec=5s + + [Install] + WantedBy=timers.target + + commands: + 01_init_service: + command: !If + - MetricsEnabled + - | + chmod +x /usr/local/bin/push-docker-metrics + systemctl daemon-reload + systemctl enable --now docker-metrics.timer + - | + chmod +x /usr/local/bin/push-docker-metrics + systemctl daemon-reload + systemctl disable --now docker-metrics.timer || true + dns_updater: packages: yum: @@ -937,14 +1012,29 @@ Resources: owner: root group: root - services: - systemd: - cfn-hup: - enabled: true - ensureRunning: true - files: - - /etc/cfn/cfn-hup.conf - - /etc/cfn/hooks.d/amazon-cloudwatch-agent-auto-reloader.conf + "/lib/systemd/system/cfn-hup.service": + content: | + [Unit] + Description=CloudFormation helper daemon + After=network.target + + [Service] + Type=simple + ExecStart=/opt/aws/bin/cfn-hup --no-daemon --verbose + Restart=always + RestartSec=10 + + [Install] + WantedBy=multi-user.target + mode: "000644" + owner: root + group: root + + commands: + 01_enable_cfn_hup: + command: | + systemctl daemon-reload + systemctl enable --now cfn-hup Properties: InstanceType: !Ref InstanceType @@ -1459,3 +1549,24 @@ Resources: Value: "1000" - Name: proc_name Value: "_timescaledb_functions.policy_compression" + + ContainerMemoryUtilizationAlarm: + Type: AWS::CloudWatch::Alarm + Condition: MetricsEnabled + Properties: + AlarmDescription: Docker container memory usage exceeded 85% of container limit + Namespace: Custom/Docker + MetricName: MaxContainerMemoryPercent + Statistic: Maximum + Period: 60 + EvaluationPeriods: 2 + ComparisonOperator: GreaterThanThreshold + Threshold: 85 + TreatMissingData: notBreaching + AlarmActions: + - !Ref SnsTopic + OKActions: + - !Ref SnsTopic + Dimensions: + - Name: InstanceName + Value: !Ref Host From b7e02cb8f974b215ec09613148fde3c942637786 Mon Sep 17 00:00:00 2001 From: Dennis Kuijs Date: Fri, 20 Nov 2026 11:59:35 +0100 Subject: [PATCH 2/6] Revert cfn-hup changes --- .ci_cd/aws/cloudformation-create-ec2.yml | 35 +++++++----------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/.ci_cd/aws/cloudformation-create-ec2.yml b/.ci_cd/aws/cloudformation-create-ec2.yml index 6f898a54e4..61e8d80c5e 100644 --- a/.ci_cd/aws/cloudformation-create-ec2.yml +++ b/.ci_cd/aws/cloudformation-create-ec2.yml @@ -1001,40 +1001,25 @@ Resources: owner: root group: root - "/etc/cfn/hooks.d/amazon-cloudwatch-agent-auto-reloader.conf": + "/etc/cfn/hooks.d/cfn-auto-reloader.conf": content: !Sub | [cfn-auto-reloader-hook] triggers=post.update path=Resources.EC2Instance.Metadata.AWS::CloudFormation::Init action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} --configsets default runas=root - mode: "000400" - owner: root - group: root - - "/lib/systemd/system/cfn-hup.service": - content: | - [Unit] - Description=CloudFormation helper daemon - After=network.target - - [Service] - Type=simple - ExecStart=/opt/aws/bin/cfn-hup --no-daemon --verbose - Restart=always - RestartSec=10 - - [Install] - WantedBy=multi-user.target - mode: "000644" + mode: 000400 owner: root group: root - commands: - 01_enable_cfn_hup: - command: | - systemctl daemon-reload - systemctl enable --now cfn-hup + services: + systemd: + cfn-hup: + enabled: true + ensureRunning: true + files: + - /etc/cfn/cfn-hup.conf + - /etc/cfn/hooks.d/cfn-auto-reloader.conf Properties: InstanceType: !Ref InstanceType From c87972a4310dcf6f31e669c83d463b1463c9b2c1 Mon Sep 17 00:00:00 2001 From: Rich Turner <7072278+richturner@users.noreply.github.com> Date: Fri, 20 Nov 2026 13:05:14 +0000 Subject: [PATCH 3/6] Measure container memory without page cache and alarm on OOM kills and restarts docker stats MemPerc includes active page cache and shared memory, so the PostgreSQL container, whose shared_buffers and cached data files fill its limit by design, kept the alarm above 85% while healthy. Read anonymous memory from each container's cgroup instead and publish it per container (ContainerMemoryPercent) as well as the instance maximum (MaxContainerMemoryPercent) used by the existing alarm, which now needs 5 minutes above the threshold. Add ContainerOomKills and ContainerRestarts, published as the number seen since the previous run, with alarms on any occurrence. An OOM kill of a container's main process (e.g. a JVM) restarts the container and resets its cgroup counters and OOMKilled flag, so it is only visible as a restart. --- .ci_cd/aws/cloudformation-create-ec2.yml | 133 ++++++++++++++++++++--- 1 file changed, 117 insertions(+), 16 deletions(-) diff --git a/.ci_cd/aws/cloudformation-create-ec2.yml b/.ci_cd/aws/cloudformation-create-ec2.yml index 61e8d80c5e..9703adb8b0 100644 --- a/.ci_cd/aws/cloudformation-create-ec2.yml +++ b/.ci_cd/aws/cloudformation-create-ec2.yml @@ -660,34 +660,93 @@ Resources: # ------------------------------------------------------------------------------------------------------ # - # Script for pushing max Docker container memory percentage to CloudWatch + # Script for pushing Docker container memory metrics to CloudWatch: + # - ContainerMemoryPercent (per container) and MaxContainerMemoryPercent: anonymous memory as a + # percentage of the container memory limit. docker stats is not used as it includes active page cache + # and shared memory, which for PostgreSQL sits close to the limit by design (shared_buffers plus + # cached data files) and is reclaimable. Containers without a memory limit are skipped. + # - ContainerOomKills: processes killed by the OOM killer inside a container since the last run. + # - ContainerRestarts: container restarts since the last run. An OOM kill of a container's main process + # (e.g. a JVM) restarts the container and resets its cgroup counters, so it is reported here instead. # # ------------------------------------------------------------------------------------------------------ NAMESPACE="Custom/Docker" + # Counters from the previous run, one " " line per container + STATE_FILE=/run/docker-metrics/counters + mkdir -p "$(dirname "$STATE_FILE")" + touch "$STATE_FILE" + + METRIC_DATA="" + SEP="" + NEW_STATE="" + MAX_PCT="" + OOM_KILLS=0 + RESTARTS=0 + + # add_metric [container name] + add_metric() { + local DIMENSIONS='{"Name":"InstanceName","Value":"${Host}"}' + if [ $# -gt 3 ]; then + DIMENSIONS="$DIMENSIONS,{\"Name\":\"Container\",\"Value\":\"$4\"}" + fi + METRIC_DATA="$METRIC_DATA$SEP{\"MetricName\":\"$1\",\"Value\":$2,\"Unit\":\"$3\",\"Dimensions\":[$DIMENSIONS]}" + SEP="," + } - MAX_MEM_PCT=$(docker stats --no-stream --format "{{.MemPerc}}" \ - | tr -d '%' \ - | grep -E '^[0-9]+(\.[0-9]+)?$' \ - | sort -nr \ - | head -n 1 || true) + while read -r ID NAME RESTART_COUNT; do + CGROUP=/sys/fs/cgroup/system.slice/docker-$ID.scope + [ -d "$CGROUP" ] || CGROUP=/sys/fs/cgroup/docker/$ID + OOM_KILL="" + + # The cgroup only exists while the container is running + if [ -f "$CGROUP/memory.stat" ]; then + OOM_KILL=$(awk '$1 == "oom_kill" {print $2}' "$CGROUP/memory.events" 2>/dev/null || true) + LIMIT=$(cat "$CGROUP/memory.max" 2>/dev/null || echo max) + ANON=$(awk '$1 == "anon" {print $2}' "$CGROUP/memory.stat" 2>/dev/null || true) + if [ "$LIMIT" != "max" ] && [ -n "$ANON" ]; then + PCT=$(awk -v a="$ANON" -v m="$LIMIT" 'BEGIN {printf "%.1f", 100 * a / m}') + add_metric ContainerMemoryPercent "$PCT" Percent "$NAME" + if [ -z "$MAX_PCT" ] || awk -v a="$PCT" -v b="$MAX_PCT" 'BEGIN {exit !(a > b)}'; then + MAX_PCT=$PCT + fi + fi + fi + [ -n "$OOM_KILL" ] || OOM_KILL=0 + + # A container seen for the first time is the baseline; a counter lower than last time has been reset + # (the cgroup is recreated on restart), so all of its current value is new + read -r PREV_OOM_KILL PREV_RESTART_COUNT <<< "$(awk -v id="$ID" '$1 == id {print $2, $3}' "$STATE_FILE")" || true + [ -n "$PREV_OOM_KILL" ] || PREV_OOM_KILL=$OOM_KILL + [ -n "$PREV_RESTART_COUNT" ] || PREV_RESTART_COUNT=$RESTART_COUNT + if [ "$OOM_KILL" -ge "$PREV_OOM_KILL" ]; then + OOM_KILLS=$((OOM_KILLS + OOM_KILL - PREV_OOM_KILL)) + else + OOM_KILLS=$((OOM_KILLS + OOM_KILL)) + fi + if [ "$RESTART_COUNT" -ge "$PREV_RESTART_COUNT" ]; then + RESTARTS=$((RESTARTS + RESTART_COUNT - PREV_RESTART_COUNT)) + fi - if [ -z "$MAX_MEM_PCT" ]; then - exit 0 - fi + NEW_STATE="$NEW_STATE$ID $OOM_KILL $RESTART_COUNT"$'\n' + done < <(docker ps -aq --no-trunc | xargs -r docker inspect --format '{{.Id}} {{.Name}} {{.RestartCount}}' | sed 's# /# #') + + [ -z "$MAX_PCT" ] || add_metric MaxContainerMemoryPercent "$MAX_PCT" Percent + add_metric ContainerOomKills "$OOM_KILLS" Count + add_metric ContainerRestarts "$RESTARTS" Count aws cloudwatch put-metric-data \ --region "${AWS::Region}" \ --namespace "$NAMESPACE" \ - --metric-name "MaxContainerMemoryPercent" \ - --dimensions "InstanceName=${Host}" \ - --value "$MAX_MEM_PCT" \ - --unit Percent + --metric-data "[$METRIC_DATA]" + + # Only record the counters once they have been published, so a failed push is reported on the next run + printf '%s' "$NEW_STATE" > "$STATE_FILE" "/lib/systemd/system/docker-metrics.service": content: | [Unit] - Description=Push max Docker container memory metric to CloudWatch + Description=Push Docker container memory metrics to CloudWatch Wants=docker-metrics.timer After=docker.service @@ -1539,12 +1598,12 @@ Resources: Type: AWS::CloudWatch::Alarm Condition: MetricsEnabled Properties: - AlarmDescription: Docker container memory usage exceeded 85% of container limit + AlarmDescription: Docker container memory (excluding page cache) exceeded 85% of container limit; see ContainerMemoryPercent for the container Namespace: Custom/Docker MetricName: MaxContainerMemoryPercent Statistic: Maximum Period: 60 - EvaluationPeriods: 2 + EvaluationPeriods: 5 ComparisonOperator: GreaterThanThreshold Threshold: 85 TreatMissingData: notBreaching @@ -1555,3 +1614,45 @@ Resources: Dimensions: - Name: InstanceName Value: !Ref Host + + ContainerOomKillAlarm: + Type: AWS::CloudWatch::Alarm + Condition: MetricsEnabled + Properties: + AlarmDescription: A process inside a Docker container was killed by the OOM killer + Namespace: Custom/Docker + MetricName: ContainerOomKills + Statistic: Sum + Period: 60 + EvaluationPeriods: 1 + ComparisonOperator: GreaterThanThreshold + Threshold: 0 + TreatMissingData: notBreaching + AlarmActions: + - !Ref SnsTopic + OKActions: + - !Ref SnsTopic + Dimensions: + - Name: InstanceName + Value: !Ref Host + + ContainerRestartAlarm: + Type: AWS::CloudWatch::Alarm + Condition: MetricsEnabled + Properties: + AlarmDescription: A Docker container restarted, e.g. its main process crashed or was killed by the OOM killer + Namespace: Custom/Docker + MetricName: ContainerRestarts + Statistic: Sum + Period: 60 + EvaluationPeriods: 1 + ComparisonOperator: GreaterThanThreshold + Threshold: 0 + TreatMissingData: notBreaching + AlarmActions: + - !Ref SnsTopic + OKActions: + - !Ref SnsTopic + Dimensions: + - Name: InstanceName + Value: !Ref Host From 6822458ce32a10d63d90815456361d1be7f152c5 Mon Sep 17 00:00:00 2001 From: Rich Turner <7072278+richturner@users.noreply.github.com> Date: Fri, 20 Nov 2026 13:14:15 +0000 Subject: [PATCH 4/6] Update cloudformation-create-ec2.yml --- .ci_cd/aws/cloudformation-create-ec2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_cd/aws/cloudformation-create-ec2.yml b/.ci_cd/aws/cloudformation-create-ec2.yml index 9703adb8b0..9f807a9399 100644 --- a/.ci_cd/aws/cloudformation-create-ec2.yml +++ b/.ci_cd/aws/cloudformation-create-ec2.yml @@ -1067,7 +1067,7 @@ Resources: path=Resources.EC2Instance.Metadata.AWS::CloudFormation::Init action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource EC2Instance --region ${AWS::Region} --configsets default runas=root - mode: 000400 + mode: "000400" owner: root group: root From 8d7c8f67bf464300d73f712a41222d890e8db095 Mon Sep 17 00:00:00 2001 From: Rich Turner <7072278+richturner@users.noreply.github.com> Date: Fri, 20 Nov 2026 13:22:43 +0000 Subject: [PATCH 5/6] Require sustained breaches for noisy alarms and remove TableBloatCountAlarm Several alarms fired on a single 5-minute datapoint for conditions that are normal when brief: - DeadTupleAlarm: dead tuples build up between autovacuum runs, so require 12 periods (1 hour) above 30% instead of one period above 20%, which matched the default autovacuum trigger and fired before every vacuum - LongRunningQueriesAlarm: 3 periods, so the nightly export and compression jobs don't trip it with a single slow query - PostgreSQLConnectionsAlarm: 3 periods, to ignore short connection bursts - AttributeEventAlarm: band width 3 and 2 of 3 periods, so ordinary traffic changes don't trip it; missing data still breaches, so a stopped manager is still reported TableBloatCountAlarm is removed: pg_table_bloat_count counts tables over 5 MB with dead tuples above 30% of live tuples, which is the same signal DeadTupleAlarm raises per table. The metric is still collected for the dashboard. --- .ci_cd/aws/cloudformation-create-ec2.yml | 41 +++++++++--------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/.ci_cd/aws/cloudformation-create-ec2.yml b/.ci_cd/aws/cloudformation-create-ec2.yml index 9f807a9399..1d0d4c2c91 100644 --- a/.ci_cd/aws/cloudformation-create-ec2.yml +++ b/.ci_cd/aws/cloudformation-create-ec2.yml @@ -1377,11 +1377,12 @@ Resources: Type: AWS::CloudWatch::Alarm Condition: MetricsEnabled Properties: - AlarmDescription: Attribute event anomaly + AlarmDescription: Attribute event rate outside the expected band (3 standard deviations) for 2 of the last 3 5-minute periods ComparisonOperator: LessThanLowerOrGreaterThanUpperThreshold - EvaluationPeriods: 1 + EvaluationPeriods: 3 + DatapointsToAlarm: 2 Metrics: - - Expression: ANOMALY_DETECTION_BAND(m1, 2) + - Expression: ANOMALY_DETECTION_BAND(m1, 3) Id: ad1 - Id: m1 MetricStat: @@ -1404,7 +1405,7 @@ Resources: Type: AWS::CloudWatch::Alarm Condition: MetricsEnabled Properties: - AlarmDescription: Available PostgreSQL Connections less then 15% of limit + AlarmDescription: Available PostgreSQL connections less than 15% of limit for 15 minutes Metrics: - Id: e1 ReturnData: true @@ -1431,7 +1432,7 @@ Resources: Value: !Ref Host Period: 300 Stat: Average - EvaluationPeriods: 1 + EvaluationPeriods: 3 ComparisonOperator: GreaterThanThreshold Threshold: 0 AlarmActions: @@ -1443,11 +1444,12 @@ Resources: Type: AWS::CloudWatch::Alarm Condition: MetricsEnabled Properties: + AlarmDescription: PostgreSQL queries running for longer than 30 seconds for 15 minutes Namespace: CWAgent-Prometheus MetricName: pg_long_running_queries_count Statistic: Average Period: 300 - EvaluationPeriods: 1 + EvaluationPeriods: 3 ComparisonOperator: GreaterThanThreshold Threshold: 0 AlarmActions: @@ -1515,24 +1517,8 @@ Resources: - Name: InstanceName Value: !Ref Host - TableBloatCountAlarm: - Type: AWS::CloudWatch::Alarm - Condition: MetricsEnabled - Properties: - Namespace: CWAgent-Prometheus - MetricName: pg_table_bloat_count - Statistic: Average - Period: 300 - EvaluationPeriods: 1 - ComparisonOperator: GreaterThanThreshold - Threshold: 0 - AlarmActions: - - !Ref SnsTopic - OKActions: - - !Ref SnsTopic - Dimensions: - - Name: InstanceName - Value: !Ref Host + # No alarm on pg_table_bloat_count: it counts tables over 5 MB with dead tuples above 30% of live tuples, which is the + # same signal DeadTupleAlarm raises per table from pg_dead_tuple_percent. The metric is still collected for the dashboard. IndexBloatCountAlarm: Type: AWS::CloudWatch::Alarm @@ -1557,15 +1543,18 @@ Resources: Type: AWS::CloudWatch::Alarm Condition: MetricsEnabled Properties: + # Dead tuples build up between autovacuum runs (which start at 20% dead by default), so only a table staying above + # the threshold for an hour indicates autovacuum is blocked or not keeping up + AlarmDescription: A table's dead tuples exceeded 30% for an hour (tables with fewer than 10,000 dead tuples report 0) Metrics: - Id: q1 Label: pg_dead_tuple_percent ReturnData: true Expression: SELECT AVG(pg_dead_tuple_percent) FROM SCHEMA("CWAgent-Prometheus", InstanceName,table_name) GROUP BY table_name ORDER BY AVG() DESC Period: 300 - EvaluationPeriods: 1 + EvaluationPeriods: 12 ComparisonOperator: GreaterThanThreshold - Threshold: 20 + Threshold: 30 AlarmActions: - !Ref SnsTopic OKActions: From 48e87561eca3c5074ab45caa811e42cb677127d6 Mon Sep 17 00:00:00 2001 From: Dennis Kuijs Date: Fri, 20 Nov 2026 15:03:39 +0100 Subject: [PATCH 6/6] small fix --- .ci_cd/aws/cloudformation-create-ec2.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.ci_cd/aws/cloudformation-create-ec2.yml b/.ci_cd/aws/cloudformation-create-ec2.yml index 1d0d4c2c91..614adf1cbe 100644 --- a/.ci_cd/aws/cloudformation-create-ec2.yml +++ b/.ci_cd/aws/cloudformation-create-ec2.yml @@ -1498,7 +1498,7 @@ Resources: - Name: InstanceName Value: !Ref Host - ChunksNeedsCompressionAlarm: + ChunksNeedingCompressionAlarm: Type: AWS::CloudWatch::Alarm Condition: MetricsEnabled Properties: @@ -1517,9 +1517,6 @@ Resources: - Name: InstanceName Value: !Ref Host - # No alarm on pg_table_bloat_count: it counts tables over 5 MB with dead tuples above 30% of live tuples, which is the - # same signal DeadTupleAlarm raises per table from pg_dead_tuple_percent. The metric is still collected for the dashboard. - IndexBloatCountAlarm: Type: AWS::CloudWatch::Alarm Condition: MetricsEnabled