From c1cb9897ef5faaf657c5db59ad35e10ca26eec54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Szab=C3=B3=2C=20Zolt=C3=A1n?= <szabozoltan69@gmail.com>
Date: Wed, 16 Sep 2026 17:42:41 +0200
Subject: [PATCH 01/27] PER translation fixes

---
 applications/argocd/staging/applications/go-api.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/staging/applications/go-api.yaml b/applications/argocd/staging/applications/go-api.yaml
index 56d2904b..c5d33c72 100644
--- a/applications/argocd/staging/applications/go-api.yaml
+++ b/applications/argocd/staging/applications/go-api.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo
     chart: ifrcgo-helm
-    targetRevision: 0.0.2-develop.c57b926b
+    targetRevision: 0.0.2-develop.c028aad9
     helm:
       valueFiles:
         - values/traefik.yaml

From 1121946ce9434956ed174475f4825ebc0c249575 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Szab=C3=B3=2C=20Zolt=C3=A1n?= <szabozoltan69@gmail.com>
Date: Wed, 16 Sep 2026 18:10:17 +0200
Subject: [PATCH 02/27] PER translation fixes

---
 applications/argocd/staging/applications/go-api.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/staging/applications/go-api.yaml b/applications/argocd/staging/applications/go-api.yaml
index c5d33c72..a2816017 100644
--- a/applications/argocd/staging/applications/go-api.yaml
+++ b/applications/argocd/staging/applications/go-api.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo
     chart: ifrcgo-helm
-    targetRevision: 0.0.2-develop.c028aad9
+    targetRevision: 0.0.2-develop.c18d58fd
     helm:
       valueFiles:
         - values/traefik.yaml

From 7311382121aa2dcc31ca673673dbbc4a15851c61 Mon Sep 17 00:00:00 2001
From: Ranjan Shrestha <ranjan.shrestha.np@gmail.com>
Date: Fri, 18 Sep 2026 10:48:58 +0545
Subject: [PATCH 03/27] chore(monty-staging): update the target revision id

---
 applications/argocd/staging/applications/montandon-etl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/staging/applications/montandon-etl.yaml b/applications/argocd/staging/applications/montandon-etl.yaml
index b00a108e..97ca5170 100644
--- a/applications/argocd/staging/applications/montandon-etl.yaml
+++ b/applications/argocd/staging/applications/montandon-etl.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo/montandon-etl
     chart: montandon-etl-helm
-    targetRevision: 0.1.1-develop.c77a12dd
+    targetRevision: 0.1.1-develop.c810abbb
     helm:
       valueFiles:
         - values/operators.yaml

From e0ed8f2d531719465e2eadf04213a881d35b34a6 Mon Sep 17 00:00:00 2001
From: Navin Ayer <thenav56@users.noreply.github.com>
Date: Fri, 18 Sep 2026 10:50:31 +0545
Subject: [PATCH 04/27] chore/go api cleanup followup (#240)

---
 .../terraform/.terraform.lock.hcl             | 21 --------------
 base-infrastructure/terraform/imports.tf      | 28 -------------------
 .../terraform/resources/.terraform.lock.hcl   | 20 -------------
 .../terraform/resources/aks.tf                | 11 --------
 .../terraform/resources/bastion.tf            |  5 ----
 .../terraform/resources/helm-ifrcgo.tf        |  9 ------
 .../terraform/resources/providers.tf          |  4 ---
 7 files changed, 98 deletions(-)
 delete mode 100644 base-infrastructure/terraform/resources/helm-ifrcgo.tf

diff --git a/base-infrastructure/terraform/.terraform.lock.hcl b/base-infrastructure/terraform/.terraform.lock.hcl
index 63bd469a..7231e82d 100644
--- a/base-infrastructure/terraform/.terraform.lock.hcl
+++ b/base-infrastructure/terraform/.terraform.lock.hcl
@@ -62,27 +62,6 @@ provider "registry.terraform.io/hashicorp/http" {
   ]
 }
 
-provider "registry.terraform.io/hashicorp/kubernetes" {
-  version     = "2.24.0"
-  constraints = "2.24.0"
-  hashes = [
-    "h1:8Ov9r+eCpuqY9LNjG3I8vKT0hX/FkyzuDxQySZVt9i4=",
-    "h1:u9lRMCdNXcB5/WQTZVMvGhNliW2pKOzj3SOVbu9yPpg=",
-    "zh:0ed83ec390a7e75c4990ebce698f14234de2b6204ed9a01cd042bb7ea5f26564",
-    "zh:195150e4fdab259c70088528006f4604557a051e037ebe8de64e92840f27e40a",
-    "zh:1a334af55f7a74adf033eb871c9fe7e9e648b41ab84321114ef4ca0e7a34fba6",
-    "zh:1ef68c3832691de21a61bf1a4e268123f3e08850712eda0b893cac908a0d1bc1",
-    "zh:44a1c58e5a6646e62b0bad653319c245f3b635dd03554dea2707a38f553e4a52",
-    "zh:54b5b374c4386f7f05b3fe986f9cb57bde4beab3bdf6ee33444f2b9a81b8af64",
-    "zh:aa8c2687ab784b72f8cdad8d3c3673dea83b33561e7b3f2d287ef0d06ff2a9e5",
-    "zh:e6ecba0503052ef3ad49ad56e17b2a73d9b55e30fcb82b040189d281e25e1a3b",
-    "zh:f105393f6487d3eb1f1636ba42d10c82950ddfef852244c1bca8d526fa23a9a3",
-    "zh:f17a8f1914ec66d80ccacecd40123362cf093abee3d3aa1ff9f8f687d8736f85",
-    "zh:f394b12ef01fa0bdf666a43ad152eb3890134f35e635ea056b18771c292de46e",
-    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
-  ]
-}
-
 provider "registry.terraform.io/hashicorp/random" {
   version     = "3.5.1"
   constraints = "~> 3.5.1"
diff --git a/base-infrastructure/terraform/imports.tf b/base-infrastructure/terraform/imports.tf
index 07079a3f..e69de29b 100644
--- a/base-infrastructure/terraform/imports.tf
+++ b/base-infrastructure/terraform/imports.tf
@@ -1,28 +0,0 @@
-# One-off adoption of resources that predate this configuration.
-#
-# The go-api workload identity was created outside Terraform for the legacy
-# `go-api-<environment>-workload-sa` federation, under the same name
-# module.go_api_resources derives from app_name and environment, so the first
-# apply of that module fails with "resource already exists".
-#
-# The identity holds no role assignments, and the federated credential the legacy
-# deployment uses stays unmanaged: the module adds a second credential rather than
-# replacing it, so both service accounts keep working during the parallel run.
-#
-# NOTE: Terraform owns the identity once this is imported, so destroying
-# module.go_api_resources also removes the legacy credential along with its parent.
-#
-# TODO: Clear this file once staging and production have both been applied.
-
-locals {
-  # Every environment in this configuration lives in the same subscription.
-  subscription_id = "39308fb0-9929-4b29-aafa-b3c78a8b0658"
-
-  # Mirrors azurerm_user_assigned_identity.workload in ./app_resources/iam.tf
-  go_api_workload_identity_name = "${replace(title("go-api"), "-", "")}${title(var.environment)}WorkloadIdentity"
-}
-
-import {
-  to = module.go_api_resources.azurerm_user_assigned_identity.workload
-  id = "/subscriptions/${local.subscription_id}/resourceGroups/${module.resources.resource_group}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${local.go_api_workload_identity_name}"
-}
diff --git a/base-infrastructure/terraform/resources/.terraform.lock.hcl b/base-infrastructure/terraform/resources/.terraform.lock.hcl
index f5131290..7231e82d 100644
--- a/base-infrastructure/terraform/resources/.terraform.lock.hcl
+++ b/base-infrastructure/terraform/resources/.terraform.lock.hcl
@@ -62,26 +62,6 @@ provider "registry.terraform.io/hashicorp/http" {
   ]
 }
 
-provider "registry.terraform.io/hashicorp/kubernetes" {
-  version     = "2.24.0"
-  constraints = "2.24.0"
-  hashes = [
-    "h1:u9lRMCdNXcB5/WQTZVMvGhNliW2pKOzj3SOVbu9yPpg=",
-    "zh:0ed83ec390a7e75c4990ebce698f14234de2b6204ed9a01cd042bb7ea5f26564",
-    "zh:195150e4fdab259c70088528006f4604557a051e037ebe8de64e92840f27e40a",
-    "zh:1a334af55f7a74adf033eb871c9fe7e9e648b41ab84321114ef4ca0e7a34fba6",
-    "zh:1ef68c3832691de21a61bf1a4e268123f3e08850712eda0b893cac908a0d1bc1",
-    "zh:44a1c58e5a6646e62b0bad653319c245f3b635dd03554dea2707a38f553e4a52",
-    "zh:54b5b374c4386f7f05b3fe986f9cb57bde4beab3bdf6ee33444f2b9a81b8af64",
-    "zh:aa8c2687ab784b72f8cdad8d3c3673dea83b33561e7b3f2d287ef0d06ff2a9e5",
-    "zh:e6ecba0503052ef3ad49ad56e17b2a73d9b55e30fcb82b040189d281e25e1a3b",
-    "zh:f105393f6487d3eb1f1636ba42d10c82950ddfef852244c1bca8d526fa23a9a3",
-    "zh:f17a8f1914ec66d80ccacecd40123362cf093abee3d3aa1ff9f8f687d8736f85",
-    "zh:f394b12ef01fa0bdf666a43ad152eb3890134f35e635ea056b18771c292de46e",
-    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
-  ]
-}
-
 provider "registry.terraform.io/hashicorp/random" {
   version     = "3.5.1"
   constraints = "~> 3.5.1"
diff --git a/base-infrastructure/terraform/resources/aks.tf b/base-infrastructure/terraform/resources/aks.tf
index db487179..bad36c7a 100644
--- a/base-infrastructure/terraform/resources/aks.tf
+++ b/base-infrastructure/terraform/resources/aks.tf
@@ -58,14 +58,3 @@ resource "azurerm_role_assignment" "storage" {
   role_definition_name = "Storage Account Contributor"
   principal_id         = azurerm_kubernetes_cluster.ifrcgo.identity[0].principal_id
 }
-
-# Kubernetes provider. No resource in this configuration uses it, and it cannot be deleted
-# while an orphaned kubernetes_* resource is still in state. This block sits inside a child
-# module, so its provider address cannot be implied; destroying such an orphan without it
-# fails with "Provider configuration not present".
-provider "kubernetes" {
-  host                   = azurerm_kubernetes_cluster.ifrcgo.kube_config[0].host
-  client_certificate     = base64decode(azurerm_kubernetes_cluster.ifrcgo.kube_config[0].client_certificate)
-  client_key             = base64decode(azurerm_kubernetes_cluster.ifrcgo.kube_config[0].client_key)
-  cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.ifrcgo.kube_config[0].cluster_ca_certificate)
-}
diff --git a/base-infrastructure/terraform/resources/bastion.tf b/base-infrastructure/terraform/resources/bastion.tf
index 1c384dc5..0bc61bc2 100644
--- a/base-infrastructure/terraform/resources/bastion.tf
+++ b/base-infrastructure/terraform/resources/bastion.tf
@@ -5,11 +5,6 @@
 # applied via this helm_release (matching how the other cluster components — traefik,
 # argocd, cert-manager, etc. — are deployed).
 #
-# TODO: An older copy of this bastion is still shipped by the go-api Helm chart
-# (deploy/helm/ifrcgo-helm/templates/bastion.yaml) and runs in the `default` namespace.
-# Both run in parallel for now; users should migrate to the new IP exposed by this
-# resource. The go-api copy will be removed in the upcoming go-api updates.
-#
 # NOTE: after editing anything under charts/ssh-bastion, bump the chart `version` in
 # Chart.yaml so the helm provider detects the change and redeploys.
 
diff --git a/base-infrastructure/terraform/resources/helm-ifrcgo.tf b/base-infrastructure/terraform/resources/helm-ifrcgo.tf
deleted file mode 100644
index 97be89a2..00000000
--- a/base-infrastructure/terraform/resources/helm-ifrcgo.tf
+++ /dev/null
@@ -1,9 +0,0 @@
-# Drops the `ifrcgo-helm` release from Terraform's state without uninstalling it from the
-# cluster's `default` namespace.
-removed {
-  from = helm_release.ifrcgo
-
-  lifecycle {
-    destroy = false
-  }
-}
diff --git a/base-infrastructure/terraform/resources/providers.tf b/base-infrastructure/terraform/resources/providers.tf
index 1b1e68cb..2f5f903e 100644
--- a/base-infrastructure/terraform/resources/providers.tf
+++ b/base-infrastructure/terraform/resources/providers.tf
@@ -14,10 +14,6 @@ terraform {
       source  = "hashicorp/helm"
       version = "~> 2.17.0"
     }
-    kubernetes = {
-      source  = "hashicorp/kubernetes"
-      version = "=2.24.0"
-    }
     random = {
       source  = "hashicorp/random"
       version = "~> 3.5.1"

From d3b486d7569a0f6b5f0c0dd0e4e29964e73891c8 Mon Sep 17 00:00:00 2001
From: Ranjan Shrestha <ranjan.shrestha.np@gmail.com>
Date: Fri, 18 Sep 2026 11:51:57 +0545
Subject: [PATCH 05/27] chore(monty-staging): update the target revision id

---
 applications/argocd/staging/applications/montandon-etl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/staging/applications/montandon-etl.yaml b/applications/argocd/staging/applications/montandon-etl.yaml
index 97ca5170..fba5f510 100644
--- a/applications/argocd/staging/applications/montandon-etl.yaml
+++ b/applications/argocd/staging/applications/montandon-etl.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo/montandon-etl
     chart: montandon-etl-helm
-    targetRevision: 0.1.1-develop.c810abbb
+    targetRevision: 0.1.1-develop.c01713d9
     helm:
       valueFiles:
         - values/operators.yaml

From 6d7ee6153cb97ac384855ac08a406a59fd2b10ee Mon Sep 17 00:00:00 2001
From: thenav56 <ayernavin@gmail.com>
Date: Sat, 19 Sep 2026 11:33:51 +0545
Subject: [PATCH 06/27] chore(monitoring): migrate staging log collection from
 promtail to alloy

---
 .../staging/platform/monitoring/loki.yaml     | 207 +++++++++++++++++-
 1 file changed, 196 insertions(+), 11 deletions(-)

diff --git a/applications/argocd/staging/platform/monitoring/loki.yaml b/applications/argocd/staging/platform/monitoring/loki.yaml
index 26129bdf..b4610f0c 100644
--- a/applications/argocd/staging/platform/monitoring/loki.yaml
+++ b/applications/argocd/staging/platform/monitoring/loki.yaml
@@ -114,20 +114,205 @@ spec:
           bloomGateway:
             replicas: 0
 
-    - chart: promtail
+    - chart: alloy
       repoURL: https://grafana.github.io/helm-charts
-      targetRevision: 6.16.6  # https://github.com/grafana/helm-charts/blob/main/charts/promtail/Chart.yaml
+      targetRevision: 1.12.1  # https://github.com/grafana/alloy/blob/main/operations/helm/charts/alloy/Chart.yaml
       helm:
         valuesObject:
-          # https://github.com/grafana/helm-charts/blob/main/charts/promtail/values.yaml
+          # https://github.com/grafana/alloy/blob/main/operations/helm/charts/alloy/values.yaml
+          # Collects pod logs from /var/log/pods and pushes them to Loki.
+          # The relabel rules match promtail's `kubernetes-pods` scrape job, so log streams
+          # carry the labels Loki queries and dashboards use: app, instance, component,
+          # node_name, namespace, job, pod and container.
 
-          config:
-            # publish data to loki
-            clients:
-              - url: http://monitoring-loki-gateway/loki/api/v1/push
-                tenant_id: 1
+          # The only CRD this chart ships is PodLogs, which this config does not use
+          crds:
+            create: false
 
-          daemonset:
+          controller:
+            type: daemonset
+            volumes:
+              extra:
+                - name: storage
+                  hostPath:
+                    path: /run/alloy
+                    type: DirectoryOrCreate
+
+          # Trimmed to what this config needs: `discovery.kubernetes` with role=pod. The chart
+          # default also grants cluster-wide configmap/secret reads for unused components.
+          # NOTE: Both lists must stay non-empty, the chart renders `[]` into the rule list.
+          rbac:
+            rules:
+              - apiGroups: [""]
+                resources: ["pods"]
+                verbs: ["get", "list", "watch"]
+            clusterRules:
+              - apiGroups: [""]
+                resources: ["nodes"]
+                verbs: ["get", "list", "watch"]
+
+          alloy:
+            # No anonymous usage stats to Grafana Labs
+            enableReporting: false
+
+            # Read positions are kept on the host so a restarted pod resumes instead of
+            # re-sending every log file it tails
+            storagePath: /run/alloy
+
+            mounts:
+              # Pod log files live under /var/log/pods
+              varlog: true
+              extra:
+                - name: storage
+                  mountPath: /run/alloy
+
+            configMap:
+              content: |-
+                logging {
+                  level  = "info"
+                  format = "logfmt"
+                }
+
+                // HOSTNAME is the node name, so each daemonset pod only watches its own node
+                discovery.kubernetes "pods" {
+                  role = "pod"
+
+                  selectors {
+                    role  = "pod"
+                    field = "spec.nodeName=" + sys.env("HOSTNAME")
+                  }
+                }
+
+                discovery.relabel "pod_logs" {
+                  targets = discovery.kubernetes.pods.targets
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["__meta_kubernetes_pod_controller_name"]
+                    regex         = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?"
+                    target_label  = "__tmp_controller_name"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = [
+                      "__meta_kubernetes_pod_label_app_kubernetes_io_name",
+                      "__meta_kubernetes_pod_label_app",
+                      "__tmp_controller_name",
+                      "__meta_kubernetes_pod_name",
+                    ]
+                    regex        = "^;*([^;]+)(;.*)?$"
+                    target_label = "app"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = [
+                      "__meta_kubernetes_pod_label_app_kubernetes_io_instance",
+                      "__meta_kubernetes_pod_label_instance",
+                    ]
+                    regex        = "^;*([^;]+)(;.*)?$"
+                    target_label = "instance"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = [
+                      "__meta_kubernetes_pod_label_app_kubernetes_io_component",
+                      "__meta_kubernetes_pod_label_component",
+                    ]
+                    regex        = "^;*([^;]+)(;.*)?$"
+                    target_label = "component"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["__meta_kubernetes_pod_node_name"]
+                    target_label  = "node_name"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["__meta_kubernetes_namespace"]
+                    target_label  = "namespace"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["namespace", "app"]
+                    separator     = "/"
+                    replacement   = "$1"
+                    target_label  = "job"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["__meta_kubernetes_pod_name"]
+                    target_label  = "pod"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["__meta_kubernetes_pod_container_name"]
+                    target_label  = "container"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
+                    separator     = "/"
+                    replacement   = "/var/log/pods/*$1/*.log"
+                    target_label  = "__path__"
+                  }
+
+                  // Static pods (control plane) are keyed by config hash instead of uid
+                  rule {
+                    action        = "replace"
+                    source_labels = [
+                      "__meta_kubernetes_pod_annotationpresent_kubernetes_io_config_hash",
+                      "__meta_kubernetes_pod_annotation_kubernetes_io_config_hash",
+                      "__meta_kubernetes_pod_container_name",
+                    ]
+                    regex        = "true/(.*)"
+                    separator    = "/"
+                    replacement  = "/var/log/pods/*$1/*.log"
+                    target_label = "__path__"
+                  }
+                }
+
+                local.file_match "pod_logs" {
+                  path_targets = discovery.relabel.pod_logs.output
+                }
+
+                loki.source.file "pod_logs" {
+                  targets    = local.file_match.pod_logs.targets
+                  forward_to = [loki.process.pod_logs.receiver]
+                }
+
+                loki.process "pod_logs" {
+                  stage.cri {}
+
+                  // Loki's ingester refuses entries older than max_chunk_age/2 (1h), so shipping
+                  // them only burns the ingestion rate limit. Happens whenever read positions are
+                  // gone (fresh install, node reboot clearing /run) and every file is read in full.
+                  stage.drop {
+                    older_than          = "1h"
+                    drop_counter_reason = "older_than_loki_accepts"
+                  }
+
+                  forward_to = [loki.write.default.receiver]
+                }
+
+                // publish data to loki
+                loki.write "default" {
+                  endpoint {
+                    url       = "http://monitoring-loki-gateway/loki/api/v1/push"
+                    tenant_id = "1"
+                  }
+                }
+
+          serviceMonitor:
             enabled: true
-          deployment:
-            enabled: false
+            # Prometheus only selects ServiceMonitors carrying the kube-prometheus-stack release label
+            additionalLabels:
+              release: monitoring-kube-prometheus-stack

From 31ac626340299c87cfae7bbd4559789ed1887540 Mon Sep 17 00:00:00 2001
From: thenav56 <ayernavin@gmail.com>
Date: Sat, 19 Sep 2026 12:06:06 +0545
Subject: [PATCH 07/27] chore(monitoring): migrate production log collection
 from promtail to alloy

---
 .../production/platform/monitoring/loki.yaml  | 207 +++++++++++++++++-
 docs/monitoring.md                            |   5 +-
 2 files changed, 199 insertions(+), 13 deletions(-)

diff --git a/applications/argocd/production/platform/monitoring/loki.yaml b/applications/argocd/production/platform/monitoring/loki.yaml
index 26129bdf..b4610f0c 100644
--- a/applications/argocd/production/platform/monitoring/loki.yaml
+++ b/applications/argocd/production/platform/monitoring/loki.yaml
@@ -114,20 +114,205 @@ spec:
           bloomGateway:
             replicas: 0
 
-    - chart: promtail
+    - chart: alloy
       repoURL: https://grafana.github.io/helm-charts
-      targetRevision: 6.16.6  # https://github.com/grafana/helm-charts/blob/main/charts/promtail/Chart.yaml
+      targetRevision: 1.12.1  # https://github.com/grafana/alloy/blob/main/operations/helm/charts/alloy/Chart.yaml
       helm:
         valuesObject:
-          # https://github.com/grafana/helm-charts/blob/main/charts/promtail/values.yaml
+          # https://github.com/grafana/alloy/blob/main/operations/helm/charts/alloy/values.yaml
+          # Collects pod logs from /var/log/pods and pushes them to Loki.
+          # The relabel rules match promtail's `kubernetes-pods` scrape job, so log streams
+          # carry the labels Loki queries and dashboards use: app, instance, component,
+          # node_name, namespace, job, pod and container.
 
-          config:
-            # publish data to loki
-            clients:
-              - url: http://monitoring-loki-gateway/loki/api/v1/push
-                tenant_id: 1
+          # The only CRD this chart ships is PodLogs, which this config does not use
+          crds:
+            create: false
 
-          daemonset:
+          controller:
+            type: daemonset
+            volumes:
+              extra:
+                - name: storage
+                  hostPath:
+                    path: /run/alloy
+                    type: DirectoryOrCreate
+
+          # Trimmed to what this config needs: `discovery.kubernetes` with role=pod. The chart
+          # default also grants cluster-wide configmap/secret reads for unused components.
+          # NOTE: Both lists must stay non-empty, the chart renders `[]` into the rule list.
+          rbac:
+            rules:
+              - apiGroups: [""]
+                resources: ["pods"]
+                verbs: ["get", "list", "watch"]
+            clusterRules:
+              - apiGroups: [""]
+                resources: ["nodes"]
+                verbs: ["get", "list", "watch"]
+
+          alloy:
+            # No anonymous usage stats to Grafana Labs
+            enableReporting: false
+
+            # Read positions are kept on the host so a restarted pod resumes instead of
+            # re-sending every log file it tails
+            storagePath: /run/alloy
+
+            mounts:
+              # Pod log files live under /var/log/pods
+              varlog: true
+              extra:
+                - name: storage
+                  mountPath: /run/alloy
+
+            configMap:
+              content: |-
+                logging {
+                  level  = "info"
+                  format = "logfmt"
+                }
+
+                // HOSTNAME is the node name, so each daemonset pod only watches its own node
+                discovery.kubernetes "pods" {
+                  role = "pod"
+
+                  selectors {
+                    role  = "pod"
+                    field = "spec.nodeName=" + sys.env("HOSTNAME")
+                  }
+                }
+
+                discovery.relabel "pod_logs" {
+                  targets = discovery.kubernetes.pods.targets
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["__meta_kubernetes_pod_controller_name"]
+                    regex         = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?"
+                    target_label  = "__tmp_controller_name"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = [
+                      "__meta_kubernetes_pod_label_app_kubernetes_io_name",
+                      "__meta_kubernetes_pod_label_app",
+                      "__tmp_controller_name",
+                      "__meta_kubernetes_pod_name",
+                    ]
+                    regex        = "^;*([^;]+)(;.*)?$"
+                    target_label = "app"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = [
+                      "__meta_kubernetes_pod_label_app_kubernetes_io_instance",
+                      "__meta_kubernetes_pod_label_instance",
+                    ]
+                    regex        = "^;*([^;]+)(;.*)?$"
+                    target_label = "instance"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = [
+                      "__meta_kubernetes_pod_label_app_kubernetes_io_component",
+                      "__meta_kubernetes_pod_label_component",
+                    ]
+                    regex        = "^;*([^;]+)(;.*)?$"
+                    target_label = "component"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["__meta_kubernetes_pod_node_name"]
+                    target_label  = "node_name"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["__meta_kubernetes_namespace"]
+                    target_label  = "namespace"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["namespace", "app"]
+                    separator     = "/"
+                    replacement   = "$1"
+                    target_label  = "job"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["__meta_kubernetes_pod_name"]
+                    target_label  = "pod"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["__meta_kubernetes_pod_container_name"]
+                    target_label  = "container"
+                  }
+
+                  rule {
+                    action        = "replace"
+                    source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
+                    separator     = "/"
+                    replacement   = "/var/log/pods/*$1/*.log"
+                    target_label  = "__path__"
+                  }
+
+                  // Static pods (control plane) are keyed by config hash instead of uid
+                  rule {
+                    action        = "replace"
+                    source_labels = [
+                      "__meta_kubernetes_pod_annotationpresent_kubernetes_io_config_hash",
+                      "__meta_kubernetes_pod_annotation_kubernetes_io_config_hash",
+                      "__meta_kubernetes_pod_container_name",
+                    ]
+                    regex        = "true/(.*)"
+                    separator    = "/"
+                    replacement  = "/var/log/pods/*$1/*.log"
+                    target_label = "__path__"
+                  }
+                }
+
+                local.file_match "pod_logs" {
+                  path_targets = discovery.relabel.pod_logs.output
+                }
+
+                loki.source.file "pod_logs" {
+                  targets    = local.file_match.pod_logs.targets
+                  forward_to = [loki.process.pod_logs.receiver]
+                }
+
+                loki.process "pod_logs" {
+                  stage.cri {}
+
+                  // Loki's ingester refuses entries older than max_chunk_age/2 (1h), so shipping
+                  // them only burns the ingestion rate limit. Happens whenever read positions are
+                  // gone (fresh install, node reboot clearing /run) and every file is read in full.
+                  stage.drop {
+                    older_than          = "1h"
+                    drop_counter_reason = "older_than_loki_accepts"
+                  }
+
+                  forward_to = [loki.write.default.receiver]
+                }
+
+                // publish data to loki
+                loki.write "default" {
+                  endpoint {
+                    url       = "http://monitoring-loki-gateway/loki/api/v1/push"
+                    tenant_id = "1"
+                  }
+                }
+
+          serviceMonitor:
             enabled: true
-          deployment:
-            enabled: false
+            # Prometheus only selects ServiceMonitors carrying the kube-prometheus-stack release label
+            additionalLabels:
+              release: monitoring-kube-prometheus-stack
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 3339425f..17b0c7a0 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -8,8 +8,9 @@
     - [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/)
 - [**loki**](https://github.com/grafana/loki/blob/main/production/helm/loki/Chart.yaml):
     - [Loki](https://grafana.com/oss/loki/)
-- [**promtail**](https://github.com/grafana/helm-charts/blob/main/charts/promtail/Chart.yaml):
-    - [Promtail](https://grafana.com/docs/loki/latest/send-data/promtail/)
+- [**alloy**](https://github.com/grafana/alloy/blob/main/operations/helm/charts/alloy/Chart.yaml):
+    - [Grafana Alloy](https://grafana.com/docs/alloy/latest/) - runs as a DaemonSet, tails pod logs from
+      `/var/log/pods` and pushes them to Loki
 
 ## Deployment
 The monitoring stack is deployed using ArgoCD, with the configuration located at:

From e283919759b7b07876027c6ab5d450d6fa6dee12 Mon Sep 17 00:00:00 2001
From: thenav56 <ayernavin@gmail.com>
Date: Sat, 19 Sep 2026 13:09:15 +0545
Subject: [PATCH 08/27] chore(monitoring): upgrade staging loki chart to 6.55.0

6.33.0 is the first version that renders storage_config.use_thanos_objstore, which the azure blob switch needs.
---
 applications/argocd/staging/platform/monitoring/loki.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/staging/platform/monitoring/loki.yaml b/applications/argocd/staging/platform/monitoring/loki.yaml
index b4610f0c..4a8c312d 100644
--- a/applications/argocd/staging/platform/monitoring/loki.yaml
+++ b/applications/argocd/staging/platform/monitoring/loki.yaml
@@ -20,7 +20,7 @@ spec:
   sources:
     - chart: loki
       repoURL: https://grafana.github.io/helm-charts
-      targetRevision: 6.28.0  # https://github.com/grafana/loki/blob/main/production/helm/loki/Chart.yaml
+      targetRevision: 6.55.0  # https://github.com/grafana/loki/blob/main/production/helm/loki/Chart.yaml
       helm:
         valuesObject:
           # Copied from https://grafana.com/docs/loki/latest/setup/install/helm/install-monolithic/ - Single Replica

From f9e29f0bac9848d848d2b8d03f45450f112a1e94 Mon Sep 17 00:00:00 2001
From: thenav56 <ayernavin@gmail.com>
Date: Sat, 19 Sep 2026 13:10:18 +0545
Subject: [PATCH 09/27] feat(monitoring): provision azure blob storage and
 workload identity for loki

Containers are lifecycle-tiered so long retention stays cheap. Loki reads the blobs as a federated identity bound to the monitoring-loki service account.
---
 .../terraform/resources/monitoring.tf         | 110 ++++++++++++++++++
 .../terraform/resources/output.tf             |  10 ++
 2 files changed, 120 insertions(+)
 create mode 100644 base-infrastructure/terraform/resources/monitoring.tf

diff --git a/base-infrastructure/terraform/resources/monitoring.tf b/base-infrastructure/terraform/resources/monitoring.tf
new file mode 100644
index 00000000..86ac2145
--- /dev/null
+++ b/base-infrastructure/terraform/resources/monitoring.tf
@@ -0,0 +1,110 @@
+# Monitoring -----------------------------------------
+
+locals {
+  # Pinned as `serviceAccount.name` in the loki argocd application. The chart's
+  # default name changed between 6.28 and 6.55, so the subject below sets it
+  # explicitly rather than tracking whatever the chart happens to generate.
+  loki_service_account = "monitoring-loki"
+}
+
+# Backs Loki's chunks and tsdb index. Keeping them here instead of on a cluster PVC
+# means retention is bounded by blob cost rather than disk size.
+#
+# Unlike the app storage accounts in storage.tf this one carries no random suffix:
+# the name is hardcoded in the loki argocd application, which cannot read terraform
+# outputs.
+resource "azurerm_storage_account" "monitoring" {
+  name                     = "${local.prefix}aksmonitoring"
+  resource_group_name      = data.azurerm_resource_group.ifrcgo.name
+  location                 = data.azurerm_resource_group.ifrcgo.location
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+}
+
+resource "azurerm_storage_container" "loki_chunks" {
+  name                  = "loki-chunks"
+  storage_account_name  = azurerm_storage_account.monitoring.name
+  container_access_type = "private"
+}
+
+resource "azurerm_storage_container" "loki_ruler" {
+  name                  = "loki-ruler"
+  storage_account_name  = azurerm_storage_account.monitoring.name
+  container_access_type = "private"
+}
+
+# Chunks are written once and read rarely after the first weeks, so they tier down.
+# Cool and Cold are both online, no rehydration, only a higher per-read cost.
+#
+# The prefix selects chunks alone. Loki writes them under the tenant id, which is
+# `fake` while `loki.auth_enabled` is false, and keeps the tsdb index under
+# `index/`, where every query reads it and it stays Hot. Turning auth on renames
+# the prefix and silently stops the tiering.
+#
+# The thresholds are coupled to `limits_config.retention_period` in the loki
+# argocd application, currently 4380h (~183 days). Azure bills an early-deletion
+# penalty for leaving Cool inside 30 days or Cold inside 90, and Loki's compactor
+# deletes at the retention period, so with a retention of R days the Cold
+# threshold has to land in [60, R - 90]. At R = 183 that window is [60, 93] and
+# 90 sits inside it. Shortening retention past 180 days closes the window, and
+# the Cold action has to go.
+#
+# Rule names are alphanumeric only. Hyphens pass `plan` and fail `apply`.
+resource "azurerm_storage_management_policy" "monitoring" {
+  storage_account_id = azurerm_storage_account.monitoring.id
+
+  rule {
+    name    = "lokichunkstiering"
+    enabled = true
+
+    filters {
+      blob_types   = ["blockBlob"]
+      prefix_match = ["${azurerm_storage_container.loki_chunks.name}/fake"]
+    }
+
+    actions {
+      base_blob {
+        tier_to_cool_after_days_since_modification_greater_than = 30
+        tier_to_cold_after_days_since_modification_greater_than = 90
+      }
+    }
+  }
+}
+
+# Loki authenticates to blob storage as this identity, through the AKS OIDC issuer.
+# No account key is involved: the thanos object store client falls through to
+# DefaultAzureCredential, which picks up the projected service account token.
+resource "azurerm_user_assigned_identity" "loki" {
+  name                = "Loki${title(var.environment)}WorkloadIdentity"
+  location            = data.azurerm_resource_group.ifrcgo.location
+  resource_group_name = data.azurerm_resource_group.ifrcgo.name
+}
+
+resource "azurerm_federated_identity_credential" "loki" {
+  name                = "loki-${var.environment}-blob-writer-identity"
+  audience            = ["api://AzureADTokenExchange"]
+  issuer              = azurerm_kubernetes_cluster.ifrcgo.oidc_issuer_url
+  parent_id           = azurerm_user_assigned_identity.loki.id
+  resource_group_name = data.azurerm_resource_group.ifrcgo.name
+  subject             = "system:serviceaccount:monitoring:${local.loki_service_account}"
+}
+
+resource "azurerm_role_assignment" "loki_chunks" {
+  scope                = "${azurerm_storage_account.monitoring.id}/blobServices/default/containers/${azurerm_storage_container.loki_chunks.name}"
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = azurerm_user_assigned_identity.loki.principal_id
+
+  # The identity is created in this same apply, and entra takes a moment to
+  # replicate it. Without this the first apply intermittently fails PrincipalNotFound.
+  skip_service_principal_aad_check = true
+}
+
+resource "azurerm_role_assignment" "loki_ruler" {
+  scope                = "${azurerm_storage_account.monitoring.id}/blobServices/default/containers/${azurerm_storage_container.loki_ruler.name}"
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = azurerm_user_assigned_identity.loki.principal_id
+
+  # The identity is created in this same apply, and entra takes a moment to
+  # replicate it. Without this the first apply intermittently fails PrincipalNotFound.
+  skip_service_principal_aad_check = true
+}
diff --git a/base-infrastructure/terraform/resources/output.tf b/base-infrastructure/terraform/resources/output.tf
index ef654b37..e11b5376 100644
--- a/base-infrastructure/terraform/resources/output.tf
+++ b/base-infrastructure/terraform/resources/output.tf
@@ -145,3 +145,13 @@ output "risk_module_storage_account_id" {
 output "risk_module_storage_account_name" {
   value = azurerm_storage_account.risk_module.name
 }
+
+# Monitoring ----------------------------------------
+output "monitoring_storage_account_name" {
+  value = azurerm_storage_account.monitoring.name
+}
+
+# Hardcoded into the loki argocd application as azure.workload.identity/client-id
+output "loki_workload_identity_client_id" {
+  value = azurerm_user_assigned_identity.loki.client_id
+}

From 4f6349ea5f6f25c5313a61622ac002bf7d4b5058 Mon Sep 17 00:00:00 2001
From: thenav56 <ayernavin@gmail.com>
Date: Sat, 19 Sep 2026 19:51:22 +0545
Subject: [PATCH 10/27] chore(monitoring): print loki storage account and
 client id on apply

Both were only reachable through the sensitive `resources` output, so filling
them into the argocd application meant digging them out by hand.
---
 base-infrastructure/terraform/output.tf           | 8 ++++++++
 base-infrastructure/terraform/resources/output.tf | 7 +++++++
 2 files changed, 15 insertions(+)

diff --git a/base-infrastructure/terraform/output.tf b/base-infrastructure/terraform/output.tf
index 4d1139ea..97f8e622 100644
--- a/base-infrastructure/terraform/output.tf
+++ b/base-infrastructure/terraform/output.tf
@@ -56,3 +56,11 @@ output "go_api_app_resource_details" {
     workload_id    = module.go_api_resources.workload_client_id
   }
 }
+
+output "monitoring_resource_details" {
+  value = {
+    storage_account_name = module.resources.monitoring_storage_account_name
+    storage_containers   = module.resources.loki_storage_containers
+    workload_id          = module.resources.loki_workload_identity_client_id
+  }
+}
diff --git a/base-infrastructure/terraform/resources/output.tf b/base-infrastructure/terraform/resources/output.tf
index e11b5376..66b4a219 100644
--- a/base-infrastructure/terraform/resources/output.tf
+++ b/base-infrastructure/terraform/resources/output.tf
@@ -151,6 +151,13 @@ output "monitoring_storage_account_name" {
   value = azurerm_storage_account.monitoring.name
 }
 
+output "loki_storage_containers" {
+  value = [
+    azurerm_storage_container.loki_chunks.name,
+    azurerm_storage_container.loki_ruler.name,
+  ]
+}
+
 # Hardcoded into the loki argocd application as azure.workload.identity/client-id
 output "loki_workload_identity_client_id" {
   value = azurerm_user_assigned_identity.loki.client_id

From 5ba83a63e0a04e064108c972d96f05b8e3dabe23 Mon Sep 17 00:00:00 2001
From: thenav56 <ayernavin@gmail.com>
Date: Sat, 19 Sep 2026 18:44:10 +0545
Subject: [PATCH 11/27] feat(monitoring): back staging loki with azure blob via
 thanos objstore

Drops the minio statefulset and its 2x32Gi PVCs; only the 10Gi statefulset
volume for the WAL and compactor working directory remains. Retention goes
to ~6 months.

Logs written before the cutover stay in minio and become unqueryable, and
minio's PVCs come from a volumeClaimTemplate so argocd will not prune them.
---
 .../staging/platform/monitoring/loki.yaml     | 53 +++++++++++++++----
 1 file changed, 43 insertions(+), 10 deletions(-)

diff --git a/applications/argocd/staging/platform/monitoring/loki.yaml b/applications/argocd/staging/platform/monitoring/loki.yaml
index 4a8c312d..d5b22fc0 100644
--- a/applications/argocd/staging/platform/monitoring/loki.yaml
+++ b/applications/argocd/staging/platform/monitoring/loki.yaml
@@ -40,9 +40,11 @@ spec:
               replication_factor: 1
             schemaConfig:
               configs:
+                # object_store names the thanos backend, so it matches
+                # storage.object_store.type rather than a cloud vendor
                 - from: "2024-04-01"
                   store: tsdb
-                  object_store: s3
+                  object_store: azure
                   schema: v13
                   index:
                     prefix: loki_index_
@@ -52,21 +54,53 @@ spec:
             limits_config:
               allow_structured_metadata: true
               volume_enabled: true
-              max_query_lookback: 2160h # ~3 months (TODO: Is this enough or to much?)
-              retention_period: 2160h   # ~3 months (TODO: Is this enough or to much?)
+              max_query_lookback: 4380h # ~6 months
+              retention_period: 4380h   # ~6 months
             # XXX: We need this disabled during loki init https://github.com/grafana/loki/issues/9634#issuecomment-2188215203
             #   Also, comment out the limits_config max_query_lookback and retention_period
             compactor:
               working_directory: /var/loki/data/retention
-              delete_request_store: s3
+              delete_request_store: azure
               retention_enabled: true
             ruler:
               enable_api: true
 
+            # Chunks and the tsdb index live in azure blob storage, so the only
+            # disk the statefulset needs is for the WAL and the compactor's
+            # working directory. Retention costs blob storage, not PVC size.
+            #
+            # `use_thanos_objstore` selects loki's thanos object store client,
+            # which the chart says will become the default. It reaches azure
+            # through DefaultAzureCredential, so with no account_key set it uses
+            # the workload identity token projected into the pod.
+            storage:
+              use_thanos_objstore: true
+              bucketNames:
+                chunks: loki-chunks
+                ruler: loki-ruler
+              object_store:
+                type: azure
+                azure:
+                  # terraform: module.resources.monitoring_storage_account_name
+                  account_name: ifrctgoaksmonitoring
+
           deploymentMode: SingleBinary
 
           singleBinary:
             replicas: 1
+            podLabels:
+              # Opts the pod in to the azure workload identity webhook, which
+              # injects the token DefaultAzureCredential reads
+              azure.workload.identity/use: "true"
+
+          serviceAccount:
+            # Pinned because the federated identity credential's subject in
+            # resources/monitoring.tf names it. The chart's generated default
+            # changed between 6.28 and 6.55, and a rename breaks blob auth.
+            name: monitoring-loki
+            annotations:
+              # terraform: module.resources.loki_workload_identity_client_id
+              azure.workload.identity/client-id: "aa46d00e-74c4-4c89-9ab3-8061cc8ee138"
 
           # XXX: Loki startup slow - https://github.com/grafana/loki/issues/7907#issuecomment-1445336799
           memberlist:
@@ -80,12 +114,8 @@ spec:
                 cpu: "0.1"
                 memory: 512Mi
 
-          # FIXME: Replace this with azure blob storage
           minio:
-            enabled: true
-            persistence:
-              enabled: true
-              size: 32Gi  # Cost is defined at tier level
+            enabled: false
 
           # Zero out replica counts of other deployment modes
           backend:
@@ -306,7 +336,10 @@ spec:
                 // publish data to loki
                 loki.write "default" {
                   endpoint {
-                    url       = "http://monitoring-loki-gateway/loki/api/v1/push"
+                    url = "http://monitoring-loki-gateway/loki/api/v1/push"
+                    // Ignored while loki runs with auth_enabled: false, which
+                    // files everything under the tenant `fake`. The blob lifecycle
+                    // rule in resources/monitoring.tf matches on that prefix.
                     tenant_id = "1"
                   }
                 }

From 0e8569de0d13e3392a65b00bd4e63078f22537ac Mon Sep 17 00:00:00 2001
From: thenav56 <ayernavin@gmail.com>
Date: Sat, 19 Sep 2026 18:44:10 +0545
Subject: [PATCH 12/27] docs(monitoring): describe loki's azure blob backend

---
 docs/monitoring.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/docs/monitoring.md b/docs/monitoring.md
index 17b0c7a0..e2a0f4f7 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -21,6 +21,56 @@ The monitoring stack is deployed using ArgoCD, with the configuration located at
 > [!Important]
 > All components are deployed to the `monitoring` namespace.
 
+## Loki storage
+
+> [!Note]
+> Staging only. Production Loki still runs on the in-cluster MinIO backend.
+
+On staging, Loki keeps its chunks and tsdb index in Azure Blob Storage rather than on
+a cluster PVC, so the retention period costs blob storage instead of disk. The
+statefulset keeps one small volume for the write-ahead log and the compactor's working
+directory. Retention is 4380h, about six months.
+
+The backend is Loki's [thanos object store client](https://grafana.com/docs/loki/latest/configure/)
+(`loki.storage.use_thanos_objstore`). It authenticates through
+`DefaultAzureCredential`, which picks up the workload identity token projected into
+the pod, so no storage account key is stored anywhere.
+
+Terraform owns the storage account, the two containers and the identity, in
+[`base-infrastructure/terraform/resources/monitoring.tf`](https://github.com/IFRCGo/go-deploy/blob/develop/base-infrastructure/terraform/resources/monitoring.tf).
+A blob lifecycle rule tiers chunks Hot to Cool at 30 days and Cool to Cold at 90. Both
+tiers stay online, so queries over old logs need no rehydration, but they do carry a
+per-GB retrieval charge that Hot does not. `max_query_lookback` spans the full
+retention period, so a careless dashboard range can reach into Cold.
+
+The thresholds are coupled to the retention period, and the terraform comment explains
+the arithmetic. Changing one without the other buys early-deletion penalties.
+
+### Cutting a cluster over
+
+The storage account name and the identity's client id are both hardcoded in the ArgoCD
+application, so `terraform apply` has to land first. Read the values back with:
+
+```bash
+cd base-infrastructure/terraform
+sed -i "s/ENVIRONMENT_TO_REPLACE/$TF_VAR_environment/g" main.tf   # as apply-infra.sh does
+terraform init
+terraform output -json resources \
+  | jq -r '{account: .monitoring_storage_account_name, clientId: .loki_workload_identity_client_id}'
+```
+
+Then set `loki.storage.object_store.azure.account_name` and
+`serviceAccount.annotations."azure.workload.identity/client-id"` in
+`monitoring/loki.yaml`, and only then let ArgoCD sync. Syncing with the placeholder
+still in place leaves Loki authenticating as nothing and dropping every write.
+
+Two things the cutover does not clean up:
+
+- Logs written before the switch stay in MinIO and become unqueryable.
+- The MinIO PVCs (`export-0` and `export-1`, 32Gi each) come from a statefulset
+  `volumeClaimTemplate`, so ArgoCD does not prune them. Delete them by hand or they
+  keep billing.
+
 ## Usage
 
 ### Retrieve Grafana Credentials

From 500aa0d580ca9cca734e0fbab81ca9d7948acf57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Szab=C3=B3=2C=20Zolt=C3=A1n?= <szabozoltan69@gmail.com>
Date: Mon, 21 Sep 2026 11:41:28 +0200
Subject: [PATCH 13/27] Fix Appeal ingesting

---
 applications/argocd/staging/applications/go-api.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/staging/applications/go-api.yaml b/applications/argocd/staging/applications/go-api.yaml
index a2816017..a0f0d87f 100644
--- a/applications/argocd/staging/applications/go-api.yaml
+++ b/applications/argocd/staging/applications/go-api.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo
     chart: ifrcgo-helm
-    targetRevision: 0.0.2-develop.c18d58fd
+    targetRevision: 0.0.2-develop.ceb5dbdb
     helm:
       valueFiles:
         - values/traefik.yaml

From 9e6418df7f2f5d9736ddeae07961afa2decdcbd7 Mon Sep 17 00:00:00 2001
From: thenav56 <ayernavin@gmail.com>
Date: Tue, 22 Sep 2026 11:10:06 +0545
Subject: [PATCH 14/27] feat(go-api/staging): django squash migration

---
 applications/argocd/staging/applications/go-api.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/staging/applications/go-api.yaml b/applications/argocd/staging/applications/go-api.yaml
index a0f0d87f..a2dec4d4 100644
--- a/applications/argocd/staging/applications/go-api.yaml
+++ b/applications/argocd/staging/applications/go-api.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo
     chart: ifrcgo-helm
-    targetRevision: 0.0.2-develop.ceb5dbdb
+    targetRevision: 0.0.2-develop.c5ab6938
     helm:
       valueFiles:
         - values/traefik.yaml

From 3abb8249810a26cee16366b1dbb720f331ff53a1 Mon Sep 17 00:00:00 2001
From: thenav56 <ayernavin@gmail.com>
Date: Tue, 22 Sep 2026 11:29:06 +0545
Subject: [PATCH 15/27] feat(go-api/production): django squash migration

---
 applications/argocd/production/applications/go-api.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/production/applications/go-api.yaml b/applications/argocd/production/applications/go-api.yaml
index 476fcf76..1338b07e 100644
--- a/applications/argocd/production/applications/go-api.yaml
+++ b/applications/argocd/production/applications/go-api.yaml
@@ -11,7 +11,7 @@ spec:
     repoURL: ghcr.io/ifrcgo
     chart: ifrcgo-helm
     # TODO: do we need to switch to a master-build chart tag
-    targetRevision: 0.0.2-develop.c054f0b4
+    targetRevision: 0.0.2-develop.c5ab6938
     helm:
       valueFiles:
         - values/traefik.yaml

From fb3b82abd3432cea0dc9d1882c1abdda8fb4020f Mon Sep 17 00:00:00 2001
From: thenav56 <ayernavin@gmail.com>
Date: Tue, 22 Sep 2026 13:52:29 +0545
Subject: [PATCH 16/27] feat(go-api/staging): django squash migration post
 cleanup

---
 applications/argocd/staging/applications/go-api.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/staging/applications/go-api.yaml b/applications/argocd/staging/applications/go-api.yaml
index a2dec4d4..552271ff 100644
--- a/applications/argocd/staging/applications/go-api.yaml
+++ b/applications/argocd/staging/applications/go-api.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo
     chart: ifrcgo-helm
-    targetRevision: 0.0.2-develop.c5ab6938
+    targetRevision: 0.0.2-develop.c0c4a10d
     helm:
       valueFiles:
         - values/traefik.yaml

From 6235c6b174e664c90a08bd5f95ba93987b1680dd Mon Sep 17 00:00:00 2001
From: thenav56 <ayernavin@gmail.com>
Date: Tue, 22 Sep 2026 13:59:25 +0545
Subject: [PATCH 17/27] feat(go-api/production): django squash migration post
 cleanup

---
 applications/argocd/production/applications/go-api.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/production/applications/go-api.yaml b/applications/argocd/production/applications/go-api.yaml
index 1338b07e..c7bc870d 100644
--- a/applications/argocd/production/applications/go-api.yaml
+++ b/applications/argocd/production/applications/go-api.yaml
@@ -11,7 +11,7 @@ spec:
     repoURL: ghcr.io/ifrcgo
     chart: ifrcgo-helm
     # TODO: do we need to switch to a master-build chart tag
-    targetRevision: 0.0.2-develop.c5ab6938
+    targetRevision: 0.0.2-develop.c0c4a10d
     helm:
       valueFiles:
         - values/traefik.yaml

From 6331e85ea468396814432d15c1809d3cae3e0b2b Mon Sep 17 00:00:00 2001
From: Ranjan Shrestha <ranjan.shrestha.np@gmail.com>
Date: Tue, 22 Sep 2026 15:02:34 +0545
Subject: [PATCH 18/27] chore(monty-staging): update the target revision id

---
 applications/argocd/staging/applications/montandon-etl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/staging/applications/montandon-etl.yaml b/applications/argocd/staging/applications/montandon-etl.yaml
index fba5f510..0c7c7f23 100644
--- a/applications/argocd/staging/applications/montandon-etl.yaml
+++ b/applications/argocd/staging/applications/montandon-etl.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo/montandon-etl
     chart: montandon-etl-helm
-    targetRevision: 0.1.1-develop.c01713d9
+    targetRevision: 0.1.1-develop.c53b0148
     helm:
       valueFiles:
         - values/operators.yaml

From 52fdc8409392669e0addee867dafdb0adfdb8b25 Mon Sep 17 00:00:00 2001
From: Felix <felix@developmentseed.org>
Date: Tue, 22 Sep 2026 13:39:09 +0200
Subject: [PATCH 19/27] feat: added montandon-eoapi-stats extractor to staging.
 (#242)

---
 .../applications/montandon-eoapi-stats.yaml   | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 applications/argocd/staging/applications/montandon-eoapi-stats.yaml

diff --git a/applications/argocd/staging/applications/montandon-eoapi-stats.yaml b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
new file mode 100644
index 00000000..20857c60
--- /dev/null
+++ b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
@@ -0,0 +1,22 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: montandon-eoapi-stats
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: ghcr.io/ifrcgo/montandon-eoapi-stats
+    chart: montandon-eoapi-stats-helm
+    targetRevision: 0.2.0
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: montandon-eoapi
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

From 7e513fbb014ef64c072c796b97757c2d15bca57a Mon Sep 17 00:00:00 2001
From: Felix <felix@developmentseed.org>
Date: Tue, 22 Sep 2026 14:12:36 +0200
Subject: [PATCH 20/27] fix: empty collections. (#243)

---
 .../argocd/staging/applications/montandon-eoapi-stats.yaml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/staging/applications/montandon-eoapi-stats.yaml b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
index 20857c60..67918151 100644
--- a/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
+++ b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo/montandon-eoapi-stats
     chart: montandon-eoapi-stats-helm
-    targetRevision: 0.2.0
+    targetRevision: 0.2.1
   destination:
     server: https://kubernetes.default.svc
     namespace: montandon-eoapi

From 40edefb8207d1bbbd2ff09b98d70266e9e3c4a49 Mon Sep 17 00:00:00 2001
From: Felix <felix@developmentseed.org>
Date: Tue, 22 Sep 2026 14:58:16 +0200
Subject: [PATCH 21/27] fix(eoapi-stats): configure ingress. (#244)

---
 .../argocd/staging/applications/montandon-eoapi-stats.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/applications/argocd/staging/applications/montandon-eoapi-stats.yaml b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
index 67918151..338f7058 100644
--- a/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
+++ b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
@@ -10,7 +10,12 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo/montandon-eoapi-stats
     chart: montandon-eoapi-stats-helm
-    targetRevision: 0.2.1
+    targetRevision: 0.2.2
+    helm:
+      valuesObject:
+        ingress:
+          enabled: true
+          host: montandon-eoapi-stats-stage.ifrc.org
   destination:
     server: https://kubernetes.default.svc
     namespace: montandon-eoapi

From fa46ced9e0483aeec7682bae14f358e2a2337895 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Szab=C3=B3=2C=20Zolt=C3=A1n?= <szabozoltan69@gmail.com>
Date: Tue, 22 Sep 2026 15:15:31 +0200
Subject: [PATCH 22/27] Add appeals to Brief

---
 applications/argocd/staging/applications/go-api.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/staging/applications/go-api.yaml b/applications/argocd/staging/applications/go-api.yaml
index 552271ff..97d7a813 100644
--- a/applications/argocd/staging/applications/go-api.yaml
+++ b/applications/argocd/staging/applications/go-api.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo
     chart: ifrcgo-helm
-    targetRevision: 0.0.2-develop.c0c4a10d
+    targetRevision: 0.0.2-develop.cb8e9907
     helm:
       valueFiles:
         - values/traefik.yaml

From 5139736f8018ee8a33daacf59b880c3b582f5c04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Szab=C3=B3=2C=20Zolt=C3=A1n?= <szabozoltan69@gmail.com>
Date: Tue, 22 Sep 2026 15:20:49 +0200
Subject: [PATCH 23/27] Add Appeals to Brief

---
 applications/argocd/production/applications/go-api.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/production/applications/go-api.yaml b/applications/argocd/production/applications/go-api.yaml
index c7bc870d..889d2abd 100644
--- a/applications/argocd/production/applications/go-api.yaml
+++ b/applications/argocd/production/applications/go-api.yaml
@@ -11,7 +11,7 @@ spec:
     repoURL: ghcr.io/ifrcgo
     chart: ifrcgo-helm
     # TODO: do we need to switch to a master-build chart tag
-    targetRevision: 0.0.2-develop.c0c4a10d
+    targetRevision: 0.0.2-develop.cb8e9907
     helm:
       valueFiles:
         - values/traefik.yaml

From 7bba20bb486a2f24e2938f932301ca4cadc65952 Mon Sep 17 00:00:00 2001
From: Felix <felix@developmentseed.org>
Date: Tue, 22 Sep 2026 15:27:53 +0200
Subject: [PATCH 24/27] chore(deps): upgraded staging eoapi-stats to 0.2.3
 (#245)

---
 .../argocd/staging/applications/montandon-eoapi-stats.yaml   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/applications/argocd/staging/applications/montandon-eoapi-stats.yaml b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
index 338f7058..a96c89e3 100644
--- a/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
+++ b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
@@ -10,12 +10,13 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo/montandon-eoapi-stats
     chart: montandon-eoapi-stats-helm
-    targetRevision: 0.2.2
+    targetRevision: 0.2.3
     helm:
       valuesObject:
         ingress:
           enabled: true
-          host: montandon-eoapi-stats-stage.ifrc.org
+          host: montandon-eoapi-stage.ifrc.org
+          path: /stats
   destination:
     server: https://kubernetes.default.svc
     namespace: montandon-eoapi

From 7578da3cef75f2dd9f86011342fec647f87e5d70 Mon Sep 17 00:00:00 2001
From: Felix <felix@developmentseed.org>
Date: Tue, 22 Sep 2026 18:07:39 +0200
Subject: [PATCH 25/27] chore(deps): upgraded staging eoapi-stats to 0.2.4
 (#246)

---
 .../argocd/staging/applications/montandon-eoapi-stats.yaml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/applications/argocd/staging/applications/montandon-eoapi-stats.yaml b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
index a96c89e3..c0fde56d 100644
--- a/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
+++ b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo/montandon-eoapi-stats
     chart: montandon-eoapi-stats-helm
-    targetRevision: 0.2.3
+    targetRevision: 0.2.4
     helm:
       valuesObject:
         ingress:

From ecbf295257f187a9cf47a8bbdaf806808f7fe495 Mon Sep 17 00:00:00 2001
From: Felix <felix@developmentseed.org>
Date: Tue, 22 Sep 2026 18:30:46 +0200
Subject: [PATCH 26/27] feat: added eoapi-stats to productions. (#248)

---
 .../applications/montandon-eoapi-stats.yaml   | 28 +++++++++++++++++++
 .../applications/montandon-eoapi-stats.yaml   |  2 +-
 2 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 applications/argocd/production/applications/montandon-eoapi-stats.yaml

diff --git a/applications/argocd/production/applications/montandon-eoapi-stats.yaml b/applications/argocd/production/applications/montandon-eoapi-stats.yaml
new file mode 100644
index 00000000..19f125ab
--- /dev/null
+++ b/applications/argocd/production/applications/montandon-eoapi-stats.yaml
@@ -0,0 +1,28 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: montandon-eoapi-stats
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: ghcr.io/ifrcgo/montandon-eoapi-stats
+    chart: montandon-eoapi-stats-helm
+    targetRevision: 0.2.5
+    helm:
+      valuesObject:
+        ingress:
+          enabled: true
+          host: montandon-eoapi.ifrc.org
+          path: /stats
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: montandon-eoapi
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
diff --git a/applications/argocd/staging/applications/montandon-eoapi-stats.yaml b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
index c0fde56d..0139c5e8 100644
--- a/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
+++ b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo/montandon-eoapi-stats
     chart: montandon-eoapi-stats-helm
-    targetRevision: 0.2.4
+    targetRevision: 0.2.5
     helm:
       valuesObject:
         ingress:

From 711c0dd4becb3d47589d5847380f230178a674d5 Mon Sep 17 00:00:00 2001
From: Felix <felix@developmentseed.org>
Date: Tue, 22 Sep 2026 19:06:42 +0200
Subject: [PATCH 27/27] chore(deps): upgraded montandon-eoapi-stats to 0.2.6
 (#249)

---
 .../argocd/production/applications/montandon-eoapi-stats.yaml   | 2 +-
 .../argocd/staging/applications/montandon-eoapi-stats.yaml      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/applications/argocd/production/applications/montandon-eoapi-stats.yaml b/applications/argocd/production/applications/montandon-eoapi-stats.yaml
index 19f125ab..e4b404b4 100644
--- a/applications/argocd/production/applications/montandon-eoapi-stats.yaml
+++ b/applications/argocd/production/applications/montandon-eoapi-stats.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo/montandon-eoapi-stats
     chart: montandon-eoapi-stats-helm
-    targetRevision: 0.2.5
+    targetRevision: 0.2.6
     helm:
       valuesObject:
         ingress:
diff --git a/applications/argocd/staging/applications/montandon-eoapi-stats.yaml b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
index 0139c5e8..07ee03c5 100644
--- a/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
+++ b/applications/argocd/staging/applications/montandon-eoapi-stats.yaml
@@ -10,7 +10,7 @@ spec:
   source:
     repoURL: ghcr.io/ifrcgo/montandon-eoapi-stats
     chart: montandon-eoapi-stats-helm
-    targetRevision: 0.2.5
+    targetRevision: 0.2.6
     helm:
       valuesObject:
         ingress: