# Optional monitoring for MOF: Prometheus with MOF's alert rules, and Grafana.
# Save next to compose.yaml and start both files together (Docker Compose v2.23+):
#   docker compose -f compose.yaml -f compose.monitoring.yaml up -d
# Grafana: http://SERVER_IP:3000 (admin / admin, change it at the first sign-in), then
# Dashboards → New → Import and upload mof-grafana-dashboard.json from mofcloud.com.
# Prometheus has no public port; it reads MOF by its service name (mof:8080).
services:
  prometheus:
    image: prom/prometheus:v2.53.1
    restart: unless-stopped
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.retention.time=15d
    configs:
      - source: prometheus
        target: /etc/prometheus/prometheus.yml
      - source: mof-alerts
        target: /etc/prometheus/mof-alerts.yml
    volumes:
      - prometheus-data:/prometheus

  grafana:
    image: grafana/grafana:11.1.0
    restart: unless-stopped
    depends_on: [prometheus]
    configs:
      - source: grafana-datasource
        target: /etc/grafana/provisioning/datasources/mof.yaml
    volumes:
      - grafana-data:/var/lib/grafana
    ports:
      - "3000:3000"

configs:
  prometheus:
    content: |
      global:
        scrape_interval: 30s
        evaluation_interval: 30s
      rule_files:
        - /etc/prometheus/mof-alerts.yml
      scrape_configs:
        - job_name: mof
          metrics_path: /internal/v1/mof/metrics
          static_configs:
            - targets: ["mof:8080"]
  mof-alerts:
    content: |
      groups:
        - name: mof
          rules:
            - alert: MofDown
              expr: up{job="mof"} == 0
              for: 2m
              labels: { severity: critical }
              annotations:
                summary: MOF is not answering metrics scrapes
            - alert: MofDatabaseUnreachable
              expr: max(mof_metrics_database_up) == 0
              for: 2m
              labels: { severity: critical }
              annotations:
                summary: MOF can't read its database
            - alert: MofAccountSyncFailing
              expr: max by (account, provider, data) (mof_account_sync_failed) == 1
              for: 30m
              labels: { severity: warning }
              annotations:
                summary: "{{ $$labels.provider }} account {{ $$labels.account }}: {{ $$labels.data }} sync is failing"
            - alert: MofAccountSyncStale
              expr: (time() - max by (account, provider, data) (mof_account_sync_last_success_timestamp_seconds > 0)) > 36 * 3600
              for: 30m
              labels: { severity: warning }
              annotations:
                summary: "{{ $$labels.provider }} account {{ $$labels.account }}: {{ $$labels.data }} not synced for 36 hours"
            - alert: MofJobsDiscarded
              expr: max(sum by (instance) (mof_jobs{state="discarded"})) - max(sum by (instance) (mof_jobs{state="discarded"} offset 1h)) > 0
              labels: { severity: warning }
              annotations:
                summary: Background jobs failed for good in the last hour
            - alert: MofJobBacklog
              expr: max(sum by (instance) (mof_jobs{state=~"available|retryable"})) > 50
              for: 30m
              labels: { severity: warning }
              annotations:
                summary: "{{ $$value }} background jobs are waiting; add workers or check for a stuck one"
            - alert: MofHighErrorRate
              expr: sum(rate(mof_http_requests_total{code=~"5.."}[5m])) / sum(rate(mof_http_requests_total[5m])) > 0.05
              for: 10m
              labels: { severity: warning }
              annotations:
                summary: "More than 5% of MOF API requests fail with 5xx"
            - alert: MofLicenseExpiring
              expr: 0 < (max(mof_license_expiry_timestamp_seconds) - time()) < 14 * 86400
              labels: { severity: warning }
              annotations:
                summary: The MOF license ends within 14 days
  grafana-datasource:
    content: |
      apiVersion: 1
      datasources:
        - name: Prometheus
          uid: mof-prometheus
          type: prometheus
          access: proxy
          url: http://prometheus:9090
          isDefault: true

volumes:
  prometheus-data:
  grafana-data:
