infra-prod/deploy/prometheus/deploy.jsonnet

// Import the kube-prometheus library
local kp = (import 'kube-prometheus/main.libsonnet') + {

  // Override common values (namespace)
  values+:: {
    common+: { namespace: 'monitoring' },
  },

  // Disable the built-in Grafana component
  grafana+:: {},

  // Prometheus customizations: external URL and persistent storage
  prometheus+:: {
    prometheus+: {
      spec+: {
        externalUrl: 'https://metrics.prod.panic.haus',
        retention: '30d',
        storage: {
          volumeClaimTemplate: {
            spec: {
              accessModes: ['ReadWriteOnce'],
              resources: { requests: { storage: '20Gi' } },
              storageClassName: 'longhorn',
            },
          },
        },
      },
    },
  },

  // Alertmanager customizations: external URL
  alertmanager+:: {
    alertmanager+: {
      spec+: {
        externalUrl: 'https://alerts.prod.panic.haus',
      },
    },
  },

  // Define a helper function for creating Ingress objects
  local ingress(name, namespace, rules) = {
    apiVersion: 'networking.k8s.io/v1',
    kind: 'Ingress',
    metadata: {
      name: name,
      namespace: namespace,
      annotations: {},
    },
    spec: {
      ingressClassName: 'nginx',
      rules: rules,
      tls: [],
    },
  },

  // Ingress resources for Prometheus and Alertmanager UIs
  ingress+:: {
    'alertmanager-main': ingress(
        'alertmanager-main',
        $.values.common.namespace,
        [{
          host: 'alerts.prod.panic.haus',
          http: {
            paths: [{
              path: '/',
              pathType: 'Prefix',
              backend: {
                service: { name: 'alertmanager-main', port: { name: 'web' } },
              },
            }],
          },
        }]
      ) + {
        metadata+: {
          annotations: { 'cert-manager.io/cluster-issuer': 'letsencrypt-prod' },
        },
        spec+: {
          tls: [{
            hosts: ['alerts.prod.panic.haus'],
            secretName: 'alerts-tls',
          }],
        },
      },

    // The Prometheus ingress will route through our OAuth2 proxy below
    'prometheus-k8s': ingress(
        'prometheus-k8s',
        $.values.common.namespace,
        [{
          host: 'metrics.prod.panic.haus',
          http: {
            paths: [{
              path: '/',
              pathType: 'Prefix',
              backend: {
                // Instead of directly pointing to Prometheus, we point to the OAuth2 proxy Service
                service: { name: 'oauth2-proxy-prometheus-service', port: { number: 4180 } },
              },
            }],
          },
        }]
      ) + {
        metadata+: {
          annotations: {
            'cert-manager.io/cluster-issuer': 'letsencrypt-prod',
            'nginx.ingress.kubernetes.io/auth-signin': 'https://$host/oauth2/start?rd=$escaped_request_uri',
            'nginx.ingress.kubernetes.io/auth-url': 'https://$host/oauth2/auth',
          },
        },
        spec+: {
          tls: [{
            hosts: ['metrics.prod.panic.haus'],
            secretName: 'monitoring-tls',
          }],
        },
      },
  },

  // Deploy the OAuth2 Proxy for Prometheus
  'oauth2-proxy-prometheus-deployment': {
    apiVersion: 'apps/v1',
    kind: 'Deployment',
    metadata: {
      name: 'oauth2-proxy-prometheus',
      namespace: $.values.common.namespace,
    },
    spec: {
      replicas: 1,
      selector: { matchLabels: { app: 'oauth2-proxy-prometheus' } },
      template: {
        metadata: { labels: { app: 'oauth2-proxy-prometheus' } },
        spec: {
          containers: [
            {
              name: 'oauth2-proxy-prometheus',
              image: 'quay.io/oauth2-proxy/oauth2-proxy:v7.8.1',
              args: [
                '--provider=keycloak',
                '--client-id=prometheus',
                '--client-secret=YbuaHkmWnUnBdCj4SFDD8J19bT4gvSgZ',
                '--cookie-secret=Y3VmaXN1aGZnMDM0OTc4ZzNoNDA4cm9pZnVoanIwZzhyago=',
                '--oidc-issuer-url=https://sso.panic.haus/realms/panic-haus',
                '--cookie-domain=metrics.prod.panic.haus',
                '--email-domain=*',
                '--http-address=0.0.0.0:4180',
                '--redirect-url=https://metrics.prod.panic.haus/oauth2/callback',
                '--upstream=http://prometheus-k8s.monitoring.svc.cluster.local:9090',
                '--scope=openid',
                '--login-url=https://sso.panic.haus/realms/panic-haus/protocol/openid-connect/auth',
                '--validate-url=https://sso.panic.haus/realms/panic-haus/protocol/openid-connect/userinfo',
                '--redeem-url=https://sso.panic.haus/realms/panic-haus/protocol/openid-connect/token',
              ],
              ports: [{ containerPort: 4180, name: 'http' }],
            },
          ],
        },
      },
    },
  },

  'oauth2-proxy-prometheus-service': {
    apiVersion: 'v1',
    kind: 'Service',
    metadata: {
      name: 'oauth2-proxy-prometheus-service',
      namespace: $.values.common.namespace,
    },
    spec: {
      ports: [{ name: 'http', port: 4180, targetPort: 4180 }],
      selector: { app: 'oauth2-proxy-prometheus' },
    },
  },
};

// Assemble all manifests (kube-prometheus stack components)
{ 'setup/0namespace-namespace': kp.kubePrometheus.namespace } +
{ ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name] 
    for name in std.filter(function(name) name != 'serviceMonitor' && name != 'prometheusRule',
                            std.objectFields(kp.prometheusOperator)) } +
{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
{ 'prometheus-operator-prometheusRule': kp.prometheusOperator.prometheusRule } +
{ 'kube-prometheus-prometheusRule': kp.kubePrometheus.prometheusRule } +
{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
{ ['blackbox-exporter-' + name]: kp.blackboxExporter[name] for name in std.objectFields(kp.blackboxExporter) } +
// { ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } +  // Grafana disabled
{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
{ ['kubernetes-' + name]: kp.kubernetesControlPlane[name] for name in std.objectFields(kp.kubernetesControlPlane) } +
{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } +
{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } +
{ [name + '-ingress']: kp.ingress[name] for name in std.objectFields(kp.ingress) }