try prometheus

deploy/prometheus/ingress-alertmanager.yaml

@@ -0,0 +1,26 @@
+# ingress-alertmanager.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: alertmanager-ingress
+  namespace: monitoring
+  annotations:
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    nginx.ingress.kubernetes.io/rewrite-target: /$2 
+spec:
+  ingressClassName: nginx
+  tls:
+  - hosts:
+      - alertmanager.prod.panic.haus
+    secretName: alertmanager-tls
+  rules:
+  - host: alertmanager.prod.panic.haus
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: alertmanager-main  # service name for Alertmanager created by operator
+            port:
+              name: web              # Alertmanager service port 9093 named "web"

deploy/prometheus/ingress-prometheus.yaml

@@ -0,0 +1,26 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: prometheus-ingress
+  namespace: monitoring
+  annotations:
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    nginx.ingress.kubernetes.io/rewrite-target: /$2 
+    nginx.ingress.kubernetes.io/proxy-buffer-size: "64k"
+spec:
+  ingressClassName: nginx
+  tls:
+  - hosts:
+      - monitoring.prod.panic.haus
+    secretName: monitoring-tls
+  rules:
+  - host: monitoring.prod.panic.haus
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: prometheus-k8s   # service name created by Prometheus operator for the Prom instance
+            port:
+              name: web            # the Prometheus service uses port name "web" for 9090

deploy/prometheus/kustomization.yaml

@@ -0,0 +1,18 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  # Include your own local manifests
+  - namespace.yaml
+  - ingress-prometheus.yaml
+  - ingress-alertmanager.yaml
+  # Reference the external repository’s manifests as a remote base:
+  - https://github.com/prometheus-operator/kube-prometheus.git//manifests?ref=main
+
+patchesStrategicMerge:
+  # Any overlay patches to customize the external manifests, e.g., setting Longhorn as storage
+  - patch-prometheus.yaml
+  - patch-alertmanager.yaml
+
+# Optionally set the namespace for the external resources
+namespace: monitoring

deploy/prometheus/namespace.yaml

@@ -0,0 +1,7 @@
+# namespace.yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring
+  labels:
+    name: monitoring

deploy/prometheus/patch-alertmanager.yaml

@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Alertmanager
+metadata:
+  name: main
+spec:
+  replicas: 2
+  storage:
+    volumeClaimTemplate:
+      spec:
+        storageClassName: longhorn
+        resources:
+          requests:
+            storage: 5Gi
+  externalUrl: "https://monitoring.prod.panic.haus"

deploy/prometheus/patch-prometheus.yaml

@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+metadata:
+  name: k8s
+spec:
+  replicas: 2
+  storage:
+    volumeClaimTemplate:
+      spec:
+        storageClassName: longhorn
+        resources:
+          requests:
+            storage: 50Gi
+  externalUrl: "https://monitoring.prod.panic.haus"

deploy/prometheus/patch-rbac.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus-operator
+roleRef:
+  kind: ClusterRole
+  name: prometheus-operator
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: prometheus-operator
+    namespace: monitoring

deploy/prometheus/prometheus-operator.yaml

@@ -0,0 +1,39 @@
+# prometheus-operator.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus-operator
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: prometheus-operator
+    app.kubernetes.io/component: controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: prometheus-operator
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: prometheus-operator
+        app.kubernetes.io/component: controller
+    spec:
+      serviceAccountName: prometheus-operator
+      containers:
+      - name: operator
+        image: quay.io/prometheus-operator/prometheus-operator:v0.66.0  # Example version
+        imagePullPolicy: IfNotPresent
+        args:
+          - --address=0.0.0.0
+          - --kubelet-service=kube-system/kubelet    # adjust if needed for your environment
+          - --manage-crds=false                      # assume CRDs applied via GitOps beforehand
+        ports:
+          - name: http
+            containerPort: 8080
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+      # Node affinity not required; operator can run on any node with correct arch
+      nodeSelector:
+        kubernetes.io/os: linux            # ensure it's a Linux node