From 027c9edb6d33b163ae97569e6c06112bb8d00932 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Beatrice=20Dellac=C3=A0?= <hello@beatrice.wtf>
Date: Mon, 31 Mar 2025 10:21:56 +0200
Subject: [PATCH] try oauth2 proxy for prom

---
 deploy/prometheus/deploy.jsonnet              | 67 ++++++++++++++++++-
 .../manifests/prometheus-k8s-ingress.yaml     |  6 +-
 2 files changed, 68 insertions(+), 5 deletions(-)

diff --git a/deploy/prometheus/deploy.jsonnet b/deploy/prometheus/deploy.jsonnet
index 1ede903..0a1abce 100644
--- a/deploy/prometheus/deploy.jsonnet
+++ b/deploy/prometheus/deploy.jsonnet
@@ -82,6 +82,7 @@ local kp = (import 'kube-prometheus/main.libsonnet') + {
         },
       },
 
+    // The Prometheus ingress will route through our OAuth2 proxy below
     'prometheus-k8s': ingress(
         'prometheus-k8s',
         $.values.common.namespace,
@@ -92,14 +93,19 @@ local kp = (import 'kube-prometheus/main.libsonnet') + {
               path: '/',
               pathType: 'Prefix',
               backend: {
-                service: { name: 'prometheus-k8s', port: { name: 'web' } },
+                // Instead of directly pointing to Prometheus, we point to the OAuth2 proxy Service
+                service: { name: 'oauth2-proxy-prometheus-service', port: { number: 4180 } },
               },
             }],
           },
         }]
       ) + {
         metadata+: {
-          annotations: { 'cert-manager.io/cluster-issuer': 'letsencrypt-prod' },
+          annotations: {
+            'cert-manager.io/cluster-issuer': 'letsencrypt-prod',
+            'nginx.ingress.kubernetes.io/auth-signin': 'https://$host/oauth2/start?rd=$escaped_request_uri',
+            'nginx.ingress.kubernetes.io/auth-url': 'https://$host/oauth2/auth',
+          },
         },
         spec+: {
           tls: [{
@@ -109,6 +115,61 @@ local kp = (import 'kube-prometheus/main.libsonnet') + {
         },
       },
   },
+
+  // Deploy the OAuth2 Proxy for Prometheus
+  'oauth2-proxy-prometheus-deployment': {
+    apiVersion: 'apps/v1',
+    kind: 'Deployment',
+    metadata: {
+      name: 'oauth2-proxy-prometheus',
+      namespace: $.values.common.namespace,
+    },
+    spec: {
+      replicas: 1,
+      selector: { matchLabels: { app: 'oauth2-proxy-prometheus' } },
+      template: {
+        metadata: { labels: { app: 'oauth2-proxy-prometheus' } },
+        spec: {
+          containers: [
+            {
+              name: 'oauth2-proxy-prometheus',
+              image: 'quay.io/oauth2-proxy/oauth2-proxy:v7.8.1',
+              args: [
+                '--provider=keycloak',
+                '--client-id=prometheus',
+                '--client-secret=YbuaHkmWnUnBdCj4SFDD8J19bT4gvSgZ',
+                '--cookie-secret=Y3VmaXN1aGZnMDM0OTc4ZzNoNDA4cm9pZnVoanIwZzhyago=',
+                '--oidc-issuer-url=https://sso.panic.haus/realms/panic-haus',
+                '--cookie-domain=metrics.prod.panic.haus',
+                '--email-domain=*',
+                '--http-address=0.0.0.0:4180',
+                '--redirect-url=https://metrics.prod.panic.haus/oauth2/callback',
+                '--upstream=http://prometheus-k8s.monitoring.svc.cluster.local:9090',
+                '--scope=openid',
+                '--login-url=https://sso.panic.haus/realms/panic-haus/protocol/openid-connect/auth',
+                '--validate-url=https://sso.panic.haus/realms/panic-haus/protocol/openid-connect/userinfo',
+                '--redeem-url=https://sso.panic.haus/realms/panic-haus/protocol/openid-connect/token',
+              ],
+              ports: [{ containerPort: 4180, name: 'http' }],
+            },
+          ],
+        },
+      },
+    },
+  },
+
+  'oauth2-proxy-prometheus-service': {
+    apiVersion: 'v1',
+    kind: 'Service',
+    metadata: {
+      name: 'oauth2-proxy-prometheus-service',
+      namespace: $.values.common.namespace,
+    },
+    spec: {
+      ports: [{ name: 'http', port: 4180, targetPort: 4180 }],
+      selector: { app: 'oauth2-proxy-prometheus' },
+    },
+  },
 };
 
 // Assemble all manifests (kube-prometheus stack components)
@@ -127,4 +188,4 @@ local kp = (import 'kube-prometheus/main.libsonnet') + {
 { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
 { ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
 { ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
-{ [name + '-ingress']: kp.ingress[name] for name in std.objectFields(kp.ingress) }
+{ [name + '-ingress']: kp.ingress[name] for name in std.objectFields(kp.ingress) }
\ No newline at end of file
diff --git a/deploy/prometheus/manifests/prometheus-k8s-ingress.yaml b/deploy/prometheus/manifests/prometheus-k8s-ingress.yaml
index 5f16c6f..7648a50 100644
--- a/deploy/prometheus/manifests/prometheus-k8s-ingress.yaml
+++ b/deploy/prometheus/manifests/prometheus-k8s-ingress.yaml
@@ -3,6 +3,8 @@ kind: Ingress
 metadata:
   annotations:
     cert-manager.io/cluster-issuer: letsencrypt-prod
+    nginx.ingress.kubernetes.io/auth-signin: https://$host/oauth2/start?rd=$escaped_request_uri
+    nginx.ingress.kubernetes.io/auth-url: https://$host/oauth2/auth
   name: prometheus-k8s
   namespace: monitoring
 spec:
@@ -13,9 +15,9 @@ spec:
       paths:
       - backend:
           service:
-            name: prometheus-k8s
+            name: oauth2-proxy-prometheus-service
             port:
-              name: web
+              number: 4180
         path: /
         pathType: Prefix
   tls: