groups:
  - name: mof
    rules:
      - alert: MofDown
        expr: up{job="mof"} == 0
        for: 2m
        labels: { severity: critical }
        annotations:
          summary: MOF is not answering metrics scrapes
      - alert: MofDatabaseUnreachable
        expr: max(mof_metrics_database_up) == 0
        for: 2m
        labels: { severity: critical }
        annotations:
          summary: MOF can't read its database
      - alert: MofAccountSyncFailing
        expr: max by (account, provider, data) (mof_account_sync_failed) == 1
        for: 30m
        labels: { severity: warning }
        annotations:
          summary: "{{ $labels.provider }} account {{ $labels.account }}: {{ $labels.data }} sync is failing"
      - alert: MofAccountSyncStale
        expr: (time() - max by (account, provider, data) (mof_account_sync_last_success_timestamp_seconds > 0)) > 36 * 3600
        for: 30m
        labels: { severity: warning }
        annotations:
          summary: "{{ $labels.provider }} account {{ $labels.account }}: {{ $labels.data }} not synced for 36 hours"
      - alert: MofJobsDiscarded
        expr: max(sum by (instance) (mof_jobs{state="discarded"})) - max(sum by (instance) (mof_jobs{state="discarded"} offset 1h)) > 0
        labels: { severity: warning }
        annotations:
          summary: Background jobs failed for good in the last hour
      - alert: MofJobBacklog
        expr: max(sum by (instance) (mof_jobs{state=~"available|retryable"})) > 50
        for: 30m
        labels: { severity: warning }
        annotations:
          summary: "{{ $value }} background jobs are waiting; add workers or check for a stuck one"
      - alert: MofHighErrorRate
        expr: sum(rate(mof_http_requests_total{code=~"5.."}[5m])) / sum(rate(mof_http_requests_total[5m])) > 0.05
        for: 10m
        labels: { severity: warning }
        annotations:
          summary: "More than 5% of MOF API requests fail with 5xx"
      - alert: MofLicenseExpiring
        expr: 0 < (max(mof_license_expiry_timestamp_seconds) - time()) < 14 * 86400
        labels: { severity: warning }
        annotations:
          summary: The MOF license ends within 14 days
